aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--client/package.json2
-rw-r--r--client/src/app/+admin/users/user-edit/user-edit.ts6
-rw-r--r--client/src/app/core/server/server.service.ts5
-rw-r--r--client/src/app/shared/video/video-details.model.ts13
-rw-r--r--client/src/app/videos/+video-watch/video-watch.component.ts28
-rw-r--r--client/src/assets/player/p2p-media-loader/p2p-media-loader-plugin.ts (renamed from client/src/assets/player/p2p-media-loader-plugin.ts)67
-rw-r--r--client/src/assets/player/p2p-media-loader/segment-url-builder.ts28
-rw-r--r--client/src/assets/player/p2p-media-loader/segment-validator.ts56
-rw-r--r--client/src/assets/player/peertube-player-manager.ts45
-rw-r--r--client/src/assets/player/peertube-plugin.ts4
-rw-r--r--client/src/assets/player/peertube-videojs-typings.ts15
-rw-r--r--client/src/assets/player/utils.ts14
-rw-r--r--client/src/assets/player/videojs-components/p2p-info-button.ts11
-rw-r--r--client/src/assets/player/webtorrent/webtorrent-plugin.ts (renamed from client/src/assets/player/webtorrent-plugin.ts)29
-rw-r--r--client/src/standalone/videos/embed.ts21
-rw-r--r--client/yarn.lock31
-rw-r--r--config/default.yaml9
-rw-r--r--config/production.yaml.example9
-rw-r--r--config/test-1.yaml1
-rw-r--r--config/test-2.yaml1
-rw-r--r--config/test-3.yaml1
-rw-r--r--config/test-4.yaml1
-rw-r--r--config/test-5.yaml1
-rw-r--r--config/test-6.yaml1
-rw-r--r--config/test.yaml2
-rw-r--r--package.json1
-rwxr-xr-xscripts/i18n/create-custom-files.ts5
-rwxr-xr-xscripts/update-host.ts14
-rw-r--r--server/controllers/activitypub/client.ts15
-rw-r--r--server/controllers/api/config.ts8
-rw-r--r--server/controllers/api/videos/index.ts19
-rw-r--r--server/controllers/static.ts9
-rw-r--r--server/controllers/tracker.ts25
-rw-r--r--server/helpers/activitypub.ts5
-rw-r--r--server/helpers/core-utils.ts8
-rw-r--r--server/helpers/custom-validators/activitypub/cache-file.ts12
-rw-r--r--server/helpers/custom-validators/activitypub/videos.ts8
-rw-r--r--server/helpers/custom-validators/misc.ts5
-rw-r--r--server/helpers/ffmpeg-utils.ts32
-rw-r--r--server/helpers/video.ts4
-rw-r--r--server/initializers/checker-before-init.ts2
-rw-r--r--server/initializers/constants.ts14
-rw-r--r--server/initializers/database.ts4
-rw-r--r--server/initializers/installer.ts5
-rw-r--r--server/initializers/migrations/0330-video-streaming-playlist.ts51
-rw-r--r--server/lib/activitypub/cache-file.ts23
-rw-r--r--server/lib/activitypub/send/send-create.ts9
-rw-r--r--server/lib/activitypub/send/send-undo.ts3
-rw-r--r--server/lib/activitypub/send/send-update.ts2
-rw-r--r--server/lib/activitypub/url.ts7
-rw-r--r--server/lib/activitypub/videos.ts97
-rw-r--r--server/lib/hls.ts110
-rw-r--r--server/lib/job-queue/handlers/video-file.ts59
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts189
-rw-r--r--server/lib/video-transcoding.ts49
-rw-r--r--server/middlewares/validators/redundancy.ts33
-rw-r--r--server/models/redundancy/video-redundancy.ts139
-rw-r--r--server/models/video/video-file.ts6
-rw-r--r--server/models/video/video-format-utils.ts61
-rw-r--r--server/models/video/video-streaming-playlist.ts154
-rw-r--r--server/models/video/video.ts179
-rw-r--r--server/tests/api/check-params/config.ts3
-rw-r--r--server/tests/api/redundancy/redundancy.ts212
-rw-r--r--server/tests/api/server/config.ts6
-rw-r--r--server/tests/api/videos/index.ts1
-rw-r--r--server/tests/api/videos/video-hls.ts145
-rw-r--r--server/tests/cli/update-host.ts11
-rw-r--r--shared/models/activitypub/objects/cache-file-object.ts4
-rw-r--r--shared/models/activitypub/objects/common-objects.ts60
-rw-r--r--shared/models/server/custom-config.model.ts3
-rw-r--r--shared/models/server/server-config.model.ts8
-rw-r--r--shared/models/videos/video-streaming-playlist.model.ts12
-rw-r--r--shared/models/videos/video-streaming-playlist.type.ts3
-rw-r--r--shared/models/videos/video.model.ts5
-rw-r--r--shared/utils/index.ts2
-rw-r--r--shared/utils/requests/requests.ts13
-rw-r--r--shared/utils/server/config.ts3
-rw-r--r--shared/utils/server/servers.ts7
-rw-r--r--shared/utils/videos/video-playlists.ts21
-rw-r--r--shared/utils/videos/videos.ts13
-rw-r--r--yarn.lock64
81 files changed, 1978 insertions, 385 deletions
diff --git a/client/package.json b/client/package.json
index a455653fe..342bab00d 100644
--- a/client/package.json
+++ b/client/package.json
@@ -134,7 +134,7 @@
134 "ngx-qrcode2": "^0.0.9", 134 "ngx-qrcode2": "^0.0.9",
135 "node-sass": "^4.9.3", 135 "node-sass": "^4.9.3",
136 "npm-font-source-sans-pro": "^1.0.2", 136 "npm-font-source-sans-pro": "^1.0.2",
137 "p2p-media-loader-hlsjs": "^0.3.0", 137 "p2p-media-loader-hlsjs": "^0.4.0",
138 "path-browserify": "^1.0.0", 138 "path-browserify": "^1.0.0",
139 "primeng": "^7.0.0", 139 "primeng": "^7.0.0",
140 "process": "^0.11.10", 140 "process": "^0.11.10",
diff --git a/client/src/app/+admin/users/user-edit/user-edit.ts b/client/src/app/+admin/users/user-edit/user-edit.ts
index 0b3511e8e..021b1feb4 100644
--- a/client/src/app/+admin/users/user-edit/user-edit.ts
+++ b/client/src/app/+admin/users/user-edit/user-edit.ts
@@ -22,7 +22,9 @@ export abstract class UserEdit extends FormReactive {
22 } 22 }
23 23
24 computeQuotaWithTranscoding () { 24 computeQuotaWithTranscoding () {
25 const resolutions = this.serverService.getConfig().transcoding.enabledResolutions 25 const transcodingConfig = this.serverService.getConfig().transcoding
26
27 const resolutions = transcodingConfig.enabledResolutions
26 const higherResolution = VideoResolution.H_1080P 28 const higherResolution = VideoResolution.H_1080P
27 let multiplier = 0 29 let multiplier = 0
28 30
@@ -30,6 +32,8 @@ export abstract class UserEdit extends FormReactive {
30 multiplier += resolution / higherResolution 32 multiplier += resolution / higherResolution
31 } 33 }
32 34
35 if (transcodingConfig.hls.enabled) multiplier *= 2
36
33 return multiplier * parseInt(this.form.value['videoQuota'], 10) 37 return multiplier * parseInt(this.form.value['videoQuota'], 10)
34 } 38 }
35 39
diff --git a/client/src/app/core/server/server.service.ts b/client/src/app/core/server/server.service.ts
index 4ae72427b..c868ccdcc 100644
--- a/client/src/app/core/server/server.service.ts
+++ b/client/src/app/core/server/server.service.ts
@@ -51,7 +51,10 @@ export class ServerService {
51 requiresEmailVerification: false 51 requiresEmailVerification: false
52 }, 52 },
53 transcoding: { 53 transcoding: {
54 enabledResolutions: [] 54 enabledResolutions: [],
55 hls: {
56 enabled: false
57 }
55 }, 58 },
56 avatar: { 59 avatar: {
57 file: { 60 file: {
diff --git a/client/src/app/shared/video/video-details.model.ts b/client/src/app/shared/video/video-details.model.ts
index fa4ca7f93..f44b4138b 100644
--- a/client/src/app/shared/video/video-details.model.ts
+++ b/client/src/app/shared/video/video-details.model.ts
@@ -3,6 +3,8 @@ import { AuthUser } from '../../core'
3import { Video } from '../../shared/video/video.model' 3import { Video } from '../../shared/video/video.model'
4import { Account } from '@app/shared/account/account.model' 4import { Account } from '@app/shared/account/account.model'
5import { VideoChannel } from '@app/shared/video-channel/video-channel.model' 5import { VideoChannel } from '@app/shared/video-channel/video-channel.model'
6import { VideoStreamingPlaylist } from '../../../../../shared/models/videos/video-streaming-playlist.model'
7import { VideoStreamingPlaylistType } from '../../../../../shared/models/videos/video-streaming-playlist.type'
6 8
7export class VideoDetails extends Video implements VideoDetailsServerModel { 9export class VideoDetails extends Video implements VideoDetailsServerModel {
8 descriptionPath: string 10 descriptionPath: string
@@ -19,6 +21,10 @@ export class VideoDetails extends Video implements VideoDetailsServerModel {
19 likesPercent: number 21 likesPercent: number
20 dislikesPercent: number 22 dislikesPercent: number
21 23
24 trackerUrls: string[]
25
26 streamingPlaylists: VideoStreamingPlaylist[]
27
22 constructor (hash: VideoDetailsServerModel, translations = {}) { 28 constructor (hash: VideoDetailsServerModel, translations = {}) {
23 super(hash, translations) 29 super(hash, translations)
24 30
@@ -30,6 +36,9 @@ export class VideoDetails extends Video implements VideoDetailsServerModel {
30 this.support = hash.support 36 this.support = hash.support
31 this.commentsEnabled = hash.commentsEnabled 37 this.commentsEnabled = hash.commentsEnabled
32 38
39 this.trackerUrls = hash.trackerUrls
40 this.streamingPlaylists = hash.streamingPlaylists
41
33 this.buildLikeAndDislikePercents() 42 this.buildLikeAndDislikePercents()
34 } 43 }
35 44
@@ -53,4 +62,8 @@ export class VideoDetails extends Video implements VideoDetailsServerModel {
53 this.likesPercent = (this.likes / (this.likes + this.dislikes)) * 100 62 this.likesPercent = (this.likes / (this.likes + this.dislikes)) * 100
54 this.dislikesPercent = (this.dislikes / (this.likes + this.dislikes)) * 100 63 this.dislikesPercent = (this.dislikes / (this.likes + this.dislikes)) * 100
55 } 64 }
65
66 getHlsPlaylist () {
67 return this.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
68 }
56} 69}
diff --git a/client/src/app/videos/+video-watch/video-watch.component.ts b/client/src/app/videos/+video-watch/video-watch.component.ts
index 6e38af195..f77316712 100644
--- a/client/src/app/videos/+video-watch/video-watch.component.ts
+++ b/client/src/app/videos/+video-watch/video-watch.component.ts
@@ -23,7 +23,7 @@ import { I18n } from '@ngx-translate/i18n-polyfill'
23import { environment } from '../../../environments/environment' 23import { environment } from '../../../environments/environment'
24import { VideoCaptionService } from '@app/shared/video-caption' 24import { VideoCaptionService } from '@app/shared/video-caption'
25import { MarkdownService } from '@app/shared/renderer' 25import { MarkdownService } from '@app/shared/renderer'
26import { PeertubePlayerManager } from '../../../assets/player/peertube-player-manager' 26import { P2PMediaLoaderOptions, PeertubePlayerManager, PlayerMode, WebtorrentOptions } from '../../../assets/player/peertube-player-manager'
27 27
28@Component({ 28@Component({
29 selector: 'my-video-watch', 29 selector: 'my-video-watch',
@@ -424,15 +424,33 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
424 serverUrl: environment.apiUrl, 424 serverUrl: environment.apiUrl,
425 425
426 videoCaptions: playerCaptions 426 videoCaptions: playerCaptions
427 }, 427 }
428 }
428 429
429 webtorrent: { 430 let mode: PlayerMode
431 const hlsPlaylist = this.video.getHlsPlaylist()
432 if (hlsPlaylist) {
433 mode = 'p2p-media-loader'
434 const p2pMediaLoader = {
435 playlistUrl: hlsPlaylist.playlistUrl,
436 segmentsSha256Url: hlsPlaylist.segmentsSha256Url,
437 redundancyBaseUrls: hlsPlaylist.redundancies.map(r => r.baseUrl),
438 trackerAnnounce: this.video.trackerUrls,
430 videoFiles: this.video.files 439 videoFiles: this.video.files
431 } 440 } as P2PMediaLoaderOptions
441
442 Object.assign(options, { p2pMediaLoader })
443 } else {
444 mode = 'webtorrent'
445 const webtorrent = {
446 videoFiles: this.video.files
447 } as WebtorrentOptions
448
449 Object.assign(options, { webtorrent })
432 } 450 }
433 451
434 this.zone.runOutsideAngular(async () => { 452 this.zone.runOutsideAngular(async () => {
435 this.player = await PeertubePlayerManager.initialize('webtorrent', options) 453 this.player = await PeertubePlayerManager.initialize(mode, options)
436 this.player.on('customError', ({ err }: { err: any }) => this.handleError(err)) 454 this.player.on('customError', ({ err }: { err: any }) => this.handleError(err))
437 }) 455 })
438 456
diff --git a/client/src/assets/player/p2p-media-loader-plugin.ts b/client/src/assets/player/p2p-media-loader/p2p-media-loader-plugin.ts
index a5b20219f..f9a2707fb 100644
--- a/client/src/assets/player/p2p-media-loader-plugin.ts
+++ b/client/src/assets/player/p2p-media-loader/p2p-media-loader-plugin.ts
@@ -1,21 +1,21 @@
1// FIXME: something weird with our path definition in tsconfig and typings 1// FIXME: something weird with our path definition in tsconfig and typings
2// @ts-ignore 2// @ts-ignore
3import * as videojs from 'video.js' 3import * as videojs from 'video.js'
4import { P2PMediaLoaderPluginOptions, PlayerNetworkInfo, VideoJSComponentInterface } from './peertube-videojs-typings' 4import { P2PMediaLoaderPluginOptions, PlayerNetworkInfo, VideoJSComponentInterface } from '../peertube-videojs-typings'
5import { Engine, initHlsJsPlayer, initVideoJsContribHlsJsPlayer } from 'p2p-media-loader-hlsjs'
6import { Events } from 'p2p-media-loader-core'
5 7
6// videojs-hlsjs-plugin needs videojs in window 8// videojs-hlsjs-plugin needs videojs in window
7window['videojs'] = videojs 9window['videojs'] = videojs
8require('@streamroot/videojs-hlsjs-plugin') 10require('@streamroot/videojs-hlsjs-plugin')
9 11
10import { Engine, initVideoJsContribHlsJsPlayer } from 'p2p-media-loader-hlsjs'
11import { Events } from 'p2p-media-loader-core'
12
13const Plugin: VideoJSComponentInterface = videojs.getPlugin('plugin') 12const Plugin: VideoJSComponentInterface = videojs.getPlugin('plugin')
14class P2pMediaLoaderPlugin extends Plugin { 13class P2pMediaLoaderPlugin extends Plugin {
15 14
16 private readonly CONSTANTS = { 15 private readonly CONSTANTS = {
17 INFO_SCHEDULER: 1000 // Don't change this 16 INFO_SCHEDULER: 1000 // Don't change this
18 } 17 }
18 private readonly options: P2PMediaLoaderPluginOptions
19 19
20 private hlsjs: any // Don't type hlsjs to not bundle the module 20 private hlsjs: any // Don't type hlsjs to not bundle the module
21 private p2pEngine: Engine 21 private p2pEngine: Engine
@@ -26,16 +26,22 @@ class P2pMediaLoaderPlugin extends Plugin {
26 totalDownload: 0, 26 totalDownload: 0,
27 totalUpload: 0 27 totalUpload: 0
28 } 28 }
29 private statsHTTPBytes = {
30 pendingDownload: [] as number[],
31 pendingUpload: [] as number[],
32 totalDownload: 0,
33 totalUpload: 0
34 }
29 35
30 private networkInfoInterval: any 36 private networkInfoInterval: any
31 37
32 constructor (player: videojs.Player, options: P2PMediaLoaderPluginOptions) { 38 constructor (player: videojs.Player, options: P2PMediaLoaderPluginOptions) {
33 super(player, options) 39 super(player, options)
34 40
41 this.options = options
42
35 videojs.Html5Hlsjs.addHook('beforeinitialize', (videojsPlayer: any, hlsjs: any) => { 43 videojs.Html5Hlsjs.addHook('beforeinitialize', (videojsPlayer: any, hlsjs: any) => {
36 this.hlsjs = hlsjs 44 this.hlsjs = hlsjs
37
38 this.initialize()
39 }) 45 })
40 46
41 initVideoJsContribHlsJsPlayer(player) 47 initVideoJsContribHlsJsPlayer(player)
@@ -44,6 +50,8 @@ class P2pMediaLoaderPlugin extends Plugin {
44 type: options.type, 50 type: options.type,
45 src: options.src 51 src: options.src
46 }) 52 })
53
54 player.ready(() => this.initialize())
47 } 55 }
48 56
49 dispose () { 57 dispose () {
@@ -51,6 +59,8 @@ class P2pMediaLoaderPlugin extends Plugin {
51 } 59 }
52 60
53 private initialize () { 61 private initialize () {
62 initHlsJsPlayer(this.hlsjs)
63
54 this.p2pEngine = this.player.tech_.options_.hlsjsConfig.loader.getEngine() 64 this.p2pEngine = this.player.tech_.options_.hlsjsConfig.loader.getEngine()
55 65
56 // Avoid using constants to not import hls.hs 66 // Avoid using constants to not import hls.hs
@@ -59,38 +69,55 @@ class P2pMediaLoaderPlugin extends Plugin {
59 this.trigger('resolutionChange', { auto: this.hlsjs.autoLevelEnabled, resolutionId: data.height }) 69 this.trigger('resolutionChange', { auto: this.hlsjs.autoLevelEnabled, resolutionId: data.height })
60 }) 70 })
61 71
72 this.p2pEngine.on(Events.SegmentError, (segment, err) => {
73 console.error('Segment error.', segment, err)
74 })
75
76 this.statsP2PBytes.numPeers = 1 + this.options.redundancyBaseUrls.length
77
62 this.runStats() 78 this.runStats()
63 } 79 }
64 80
65 private runStats () { 81 private runStats () {
66 this.p2pEngine.on(Events.PieceBytesDownloaded, (method: string, size: number) => { 82 this.p2pEngine.on(Events.PieceBytesDownloaded, (method: string, size: number) => {
67 if (method === 'p2p') { 83 const elem = method === 'p2p' ? this.statsP2PBytes : this.statsHTTPBytes
68 this.statsP2PBytes.pendingDownload.push(size) 84
69 this.statsP2PBytes.totalDownload += size 85 elem.pendingDownload.push(size)
70 } 86 elem.totalDownload += size
71 }) 87 })
72 88
73 this.p2pEngine.on(Events.PieceBytesUploaded, (method: string, size: number) => { 89 this.p2pEngine.on(Events.PieceBytesUploaded, (method: string, size: number) => {
74 if (method === 'p2p') { 90 const elem = method === 'p2p' ? this.statsP2PBytes : this.statsHTTPBytes
75 this.statsP2PBytes.pendingUpload.push(size) 91
76 this.statsP2PBytes.totalUpload += size 92 elem.pendingUpload.push(size)
77 } 93 elem.totalUpload += size
78 }) 94 })
79 95
80 this.p2pEngine.on(Events.PeerConnect, () => this.statsP2PBytes.numPeers++) 96 this.p2pEngine.on(Events.PeerConnect, () => this.statsP2PBytes.numPeers++)
81 this.p2pEngine.on(Events.PeerClose, () => this.statsP2PBytes.numPeers--) 97 this.p2pEngine.on(Events.PeerClose, () => this.statsP2PBytes.numPeers--)
82 98
83 this.networkInfoInterval = setInterval(() => { 99 this.networkInfoInterval = setInterval(() => {
84 let downloadSpeed = this.statsP2PBytes.pendingDownload.reduce((a: number, b: number) => a + b, 0) 100 const p2pDownloadSpeed = this.arraySum(this.statsP2PBytes.pendingDownload)
85 let uploadSpeed = this.statsP2PBytes.pendingUpload.reduce((a: number, b: number) => a + b, 0) 101 const p2pUploadSpeed = this.arraySum(this.statsP2PBytes.pendingUpload)
102
103 const httpDownloadSpeed = this.arraySum(this.statsHTTPBytes.pendingDownload)
104 const httpUploadSpeed = this.arraySum(this.statsHTTPBytes.pendingUpload)
86 105
87 this.statsP2PBytes.pendingDownload = [] 106 this.statsP2PBytes.pendingDownload = []
88 this.statsP2PBytes.pendingUpload = [] 107 this.statsP2PBytes.pendingUpload = []
108 this.statsHTTPBytes.pendingDownload = []
109 this.statsHTTPBytes.pendingUpload = []
89 110
90 return this.player.trigger('p2pInfo', { 111 return this.player.trigger('p2pInfo', {
112 http: {
113 downloadSpeed: httpDownloadSpeed,
114 uploadSpeed: httpUploadSpeed,
115 downloaded: this.statsHTTPBytes.totalDownload,
116 uploaded: this.statsHTTPBytes.totalUpload
117 },
91 p2p: { 118 p2p: {
92 downloadSpeed, 119 downloadSpeed: p2pDownloadSpeed,
93 uploadSpeed, 120 uploadSpeed: p2pUploadSpeed,
94 numPeers: this.statsP2PBytes.numPeers, 121 numPeers: this.statsP2PBytes.numPeers,
95 downloaded: this.statsP2PBytes.totalDownload, 122 downloaded: this.statsP2PBytes.totalDownload,
96 uploaded: this.statsP2PBytes.totalUpload 123 uploaded: this.statsP2PBytes.totalUpload
@@ -98,6 +125,10 @@ class P2pMediaLoaderPlugin extends Plugin {
98 } as PlayerNetworkInfo) 125 } as PlayerNetworkInfo)
99 }, this.CONSTANTS.INFO_SCHEDULER) 126 }, this.CONSTANTS.INFO_SCHEDULER)
100 } 127 }
128
129 private arraySum (data: number[]) {
130 return data.reduce((a: number, b: number) => a + b, 0)
131 }
101} 132}
102 133
103videojs.registerPlugin('p2pMediaLoader', P2pMediaLoaderPlugin) 134videojs.registerPlugin('p2pMediaLoader', P2pMediaLoaderPlugin)
diff --git a/client/src/assets/player/p2p-media-loader/segment-url-builder.ts b/client/src/assets/player/p2p-media-loader/segment-url-builder.ts
new file mode 100644
index 000000000..32e7ce4f2
--- /dev/null
+++ b/client/src/assets/player/p2p-media-loader/segment-url-builder.ts
@@ -0,0 +1,28 @@
1import { basename } from 'path'
2import { Segment } from 'p2p-media-loader-core'
3
4function segmentUrlBuilderFactory (baseUrls: string[]) {
5 return function segmentBuilder (segment: Segment) {
6 const max = baseUrls.length + 1
7 const i = getRandomInt(max)
8
9 if (i === max - 1) return segment.url
10
11 let newBaseUrl = baseUrls[i]
12 let middlePart = newBaseUrl.endsWith('/') ? '' : '/'
13
14 return newBaseUrl + middlePart + basename(segment.url)
15 }
16}
17
18// ---------------------------------------------------------------------------
19
20export {
21 segmentUrlBuilderFactory
22}
23
24// ---------------------------------------------------------------------------
25
26function getRandomInt (max: number) {
27 return Math.floor(Math.random() * Math.floor(max))
28}
diff --git a/client/src/assets/player/p2p-media-loader/segment-validator.ts b/client/src/assets/player/p2p-media-loader/segment-validator.ts
new file mode 100644
index 000000000..8f4922daa
--- /dev/null
+++ b/client/src/assets/player/p2p-media-loader/segment-validator.ts
@@ -0,0 +1,56 @@
1import { Segment } from 'p2p-media-loader-core'
2import { basename } from 'path'
3
4function segmentValidatorFactory (segmentsSha256Url: string) {
5 const segmentsJSON = fetchSha256Segments(segmentsSha256Url)
6
7 return async function segmentValidator (segment: Segment) {
8 const segmentName = basename(segment.url)
9
10 const hashShouldBe = (await segmentsJSON)[segmentName]
11 if (hashShouldBe === undefined) {
12 throw new Error(`Unknown segment name ${segmentName} in segment validator`)
13 }
14
15 const calculatedSha = bufferToEx(await sha256(segment.data))
16 if (calculatedSha !== hashShouldBe) {
17 throw new Error(`Hashes does not correspond for segment ${segmentName} (expected: ${hashShouldBe} instead of ${calculatedSha})`)
18 }
19 }
20}
21
22// ---------------------------------------------------------------------------
23
24export {
25 segmentValidatorFactory
26}
27
28// ---------------------------------------------------------------------------
29
30function fetchSha256Segments (url: string) {
31 return fetch(url)
32 .then(res => res.json())
33 .catch(err => {
34 console.error('Cannot get sha256 segments', err)
35 return {}
36 })
37}
38
39function sha256 (data?: ArrayBuffer) {
40 if (!data) return undefined
41
42 return window.crypto.subtle.digest('SHA-256', data)
43}
44
45// Thanks: https://stackoverflow.com/a/53307879
46function bufferToEx (buffer?: ArrayBuffer) {
47 if (!buffer) return ''
48
49 let s = ''
50 const h = '0123456789abcdef'
51 const o = new Uint8Array(buffer)
52
53 o.forEach((v: any) => s += h[ v >> 4 ] + h[ v & 15 ])
54
55 return s
56}
diff --git a/client/src/assets/player/peertube-player-manager.ts b/client/src/assets/player/peertube-player-manager.ts
index 91ca6a2aa..3fdba6fdf 100644
--- a/client/src/assets/player/peertube-player-manager.ts
+++ b/client/src/assets/player/peertube-player-manager.ts
@@ -13,8 +13,10 @@ import './videojs-components/p2p-info-button'
13import './videojs-components/peertube-load-progress-bar' 13import './videojs-components/peertube-load-progress-bar'
14import './videojs-components/theater-button' 14import './videojs-components/theater-button'
15import { P2PMediaLoaderPluginOptions, UserWatching, VideoJSCaption, VideoJSPluginOptions, videojsUntyped } from './peertube-videojs-typings' 15import { P2PMediaLoaderPluginOptions, UserWatching, VideoJSCaption, VideoJSPluginOptions, videojsUntyped } from './peertube-videojs-typings'
16import { buildVideoEmbed, buildVideoLink, copyToClipboard } from './utils' 16import { buildVideoEmbed, buildVideoLink, copyToClipboard, getRtcConfig } from './utils'
17import { getCompleteLocale, getShortLocale, is18nLocale, isDefaultLocale } from '../../../../shared/models/i18n/i18n' 17import { getCompleteLocale, getShortLocale, is18nLocale, isDefaultLocale } from '../../../../shared/models/i18n/i18n'
18import { segmentValidatorFactory } from './p2p-media-loader/segment-validator'
19import { segmentUrlBuilderFactory } from './p2p-media-loader/segment-url-builder'
18 20
19// Change 'Playback Rate' to 'Speed' (smaller for our settings menu) 21// Change 'Playback Rate' to 'Speed' (smaller for our settings menu)
20videojsUntyped.getComponent('PlaybackRateMenuButton').prototype.controlText_ = 'Speed' 22videojsUntyped.getComponent('PlaybackRateMenuButton').prototype.controlText_ = 'Speed'
@@ -31,7 +33,10 @@ export type WebtorrentOptions = {
31 33
32export type P2PMediaLoaderOptions = { 34export type P2PMediaLoaderOptions = {
33 playlistUrl: string 35 playlistUrl: string
36 segmentsSha256Url: string
34 trackerAnnounce: string[] 37 trackerAnnounce: string[]
38 redundancyBaseUrls: string[]
39 videoFiles: VideoFile[]
35} 40}
36 41
37export type CommonOptions = { 42export type CommonOptions = {
@@ -90,11 +95,11 @@ export class PeertubePlayerManager {
90 static async initialize (mode: PlayerMode, options: PeertubePlayerManagerOptions) { 95 static async initialize (mode: PlayerMode, options: PeertubePlayerManagerOptions) {
91 let p2pMediaLoader: any 96 let p2pMediaLoader: any
92 97
93 if (mode === 'webtorrent') await import('./webtorrent-plugin') 98 if (mode === 'webtorrent') await import('./webtorrent/webtorrent-plugin')
94 if (mode === 'p2p-media-loader') { 99 if (mode === 'p2p-media-loader') {
95 [ p2pMediaLoader ] = await Promise.all([ 100 [ p2pMediaLoader ] = await Promise.all([
96 import('p2p-media-loader-hlsjs'), 101 import('p2p-media-loader-hlsjs'),
97 import('./p2p-media-loader-plugin') 102 import('./p2p-media-loader/p2p-media-loader-plugin')
98 ]) 103 ])
99 } 104 }
100 105
@@ -144,11 +149,14 @@ export class PeertubePlayerManager {
144 const commonOptions = options.common 149 const commonOptions = options.common
145 const webtorrentOptions = options.webtorrent 150 const webtorrentOptions = options.webtorrent
146 const p2pMediaLoaderOptions = options.p2pMediaLoader 151 const p2pMediaLoaderOptions = options.p2pMediaLoader
152
153 let autoplay = options.common.autoplay
147 let html5 = {} 154 let html5 = {}
148 155
149 const plugins: VideoJSPluginOptions = { 156 const plugins: VideoJSPluginOptions = {
150 peertube: { 157 peertube: {
151 autoplay: commonOptions.autoplay, // Use peertube plugin autoplay because we get the file by webtorrent 158 mode,
159 autoplay, // Use peertube plugin autoplay because we get the file by webtorrent
152 videoViewUrl: commonOptions.videoViewUrl, 160 videoViewUrl: commonOptions.videoViewUrl,
153 videoDuration: commonOptions.videoDuration, 161 videoDuration: commonOptions.videoDuration,
154 startTime: commonOptions.startTime, 162 startTime: commonOptions.startTime,
@@ -160,19 +168,35 @@ export class PeertubePlayerManager {
160 168
161 if (p2pMediaLoaderOptions) { 169 if (p2pMediaLoaderOptions) {
162 const p2pMediaLoader: P2PMediaLoaderPluginOptions = { 170 const p2pMediaLoader: P2PMediaLoaderPluginOptions = {
171 redundancyBaseUrls: options.p2pMediaLoader.redundancyBaseUrls,
163 type: 'application/x-mpegURL', 172 type: 'application/x-mpegURL',
164 src: p2pMediaLoaderOptions.playlistUrl 173 src: p2pMediaLoaderOptions.playlistUrl
165 } 174 }
166 175
176 const trackerAnnounce = p2pMediaLoaderOptions.trackerAnnounce
177 .filter(t => t.startsWith('ws'))
178
167 const p2pMediaLoaderConfig = { 179 const p2pMediaLoaderConfig = {
168 // loader: { 180 loader: {
169 // trackerAnnounce: p2pMediaLoaderOptions.trackerAnnounce 181 trackerAnnounce,
170 // }, 182 segmentValidator: segmentValidatorFactory(options.p2pMediaLoader.segmentsSha256Url),
183 rtcConfig: getRtcConfig(),
184 requiredSegmentsPriority: 5,
185 segmentUrlBuilder: segmentUrlBuilderFactory(options.p2pMediaLoader.redundancyBaseUrls)
186 },
171 segments: { 187 segments: {
172 swarmId: p2pMediaLoaderOptions.playlistUrl 188 swarmId: p2pMediaLoaderOptions.playlistUrl
173 } 189 }
174 } 190 }
175 const streamrootHls = { 191 const streamrootHls = {
192 levelLabelHandler: (level: { height: number, width: number }) => {
193 const file = p2pMediaLoaderOptions.videoFiles.find(f => f.resolution.id === level.height)
194
195 let label = file.resolution.label
196 if (file.fps >= 50) label += file.fps
197
198 return label
199 },
176 html5: { 200 html5: {
177 hlsjsConfig: { 201 hlsjsConfig: {
178 liveSyncDurationCount: 7, 202 liveSyncDurationCount: 7,
@@ -187,12 +211,15 @@ export class PeertubePlayerManager {
187 211
188 if (webtorrentOptions) { 212 if (webtorrentOptions) {
189 const webtorrent = { 213 const webtorrent = {
190 autoplay: commonOptions.autoplay, 214 autoplay,
191 videoDuration: commonOptions.videoDuration, 215 videoDuration: commonOptions.videoDuration,
192 playerElement: commonOptions.playerElement, 216 playerElement: commonOptions.playerElement,
193 videoFiles: webtorrentOptions.videoFiles 217 videoFiles: webtorrentOptions.videoFiles
194 } 218 }
195 Object.assign(plugins, { webtorrent }) 219 Object.assign(plugins, { webtorrent })
220
221 // WebTorrent plugin handles autoplay, because we do some hackish stuff in there
222 autoplay = false
196 } 223 }
197 224
198 const videojsOptions = { 225 const videojsOptions = {
@@ -208,7 +235,7 @@ export class PeertubePlayerManager {
208 : undefined, // Undefined so the player knows it has to check the local storage 235 : undefined, // Undefined so the player knows it has to check the local storage
209 236
210 poster: commonOptions.poster, 237 poster: commonOptions.poster,
211 autoplay: false, 238 autoplay,
212 inactivityTimeout: commonOptions.inactivityTimeout, 239 inactivityTimeout: commonOptions.inactivityTimeout,
213 playbackRates: [ 0.5, 0.75, 1, 1.25, 1.5, 2 ], 240 playbackRates: [ 0.5, 0.75, 1, 1.25, 1.5, 2 ],
214 plugins, 241 plugins,
diff --git a/client/src/assets/player/peertube-plugin.ts b/client/src/assets/player/peertube-plugin.ts
index f83d9094a..aacbf5f6e 100644
--- a/client/src/assets/player/peertube-plugin.ts
+++ b/client/src/assets/player/peertube-plugin.ts
@@ -52,12 +52,12 @@ class PeerTubePlugin extends Plugin {
52 this.player.ready(() => { 52 this.player.ready(() => {
53 const playerOptions = this.player.options_ 53 const playerOptions = this.player.options_
54 54
55 if (this.player.webtorrent) { 55 if (options.mode === 'webtorrent') {
56 this.player.webtorrent().on('resolutionChange', (_: any, d: any) => this.handleResolutionChange(d)) 56 this.player.webtorrent().on('resolutionChange', (_: any, d: any) => this.handleResolutionChange(d))
57 this.player.webtorrent().on('autoResolutionChange', (_: any, d: any) => this.trigger('autoResolutionChange', d)) 57 this.player.webtorrent().on('autoResolutionChange', (_: any, d: any) => this.trigger('autoResolutionChange', d))
58 } 58 }
59 59
60 if (this.player.p2pMediaLoader) { 60 if (options.mode === 'p2p-media-loader') {
61 this.player.p2pMediaLoader().on('resolutionChange', (_: any, d: any) => this.handleResolutionChange(d)) 61 this.player.p2pMediaLoader().on('resolutionChange', (_: any, d: any) => this.handleResolutionChange(d))
62 } 62 }
63 63
diff --git a/client/src/assets/player/peertube-videojs-typings.ts b/client/src/assets/player/peertube-videojs-typings.ts
index fff992a6f..79a5a6c4d 100644
--- a/client/src/assets/player/peertube-videojs-typings.ts
+++ b/client/src/assets/player/peertube-videojs-typings.ts
@@ -4,12 +4,15 @@ import * as videojs from 'video.js'
4 4
5import { VideoFile } from '../../../../shared/models/videos/video.model' 5import { VideoFile } from '../../../../shared/models/videos/video.model'
6import { PeerTubePlugin } from './peertube-plugin' 6import { PeerTubePlugin } from './peertube-plugin'
7import { WebTorrentPlugin } from './webtorrent-plugin' 7import { WebTorrentPlugin } from './webtorrent/webtorrent-plugin'
8import { P2pMediaLoaderPlugin } from './p2p-media-loader/p2p-media-loader-plugin'
9import { PlayerMode } from './peertube-player-manager'
8 10
9declare namespace videojs { 11declare namespace videojs {
10 interface Player { 12 interface Player {
11 peertube (): PeerTubePlugin 13 peertube (): PeerTubePlugin
12 webtorrent (): WebTorrentPlugin 14 webtorrent (): WebTorrentPlugin
15 p2pMediaLoader (): P2pMediaLoaderPlugin
13 } 16 }
14} 17}
15 18
@@ -33,6 +36,8 @@ type UserWatching = {
33} 36}
34 37
35type PeerTubePluginOptions = { 38type PeerTubePluginOptions = {
39 mode: PlayerMode
40
36 autoplay: boolean 41 autoplay: boolean
37 videoViewUrl: string 42 videoViewUrl: string
38 videoDuration: number 43 videoDuration: number
@@ -54,6 +59,7 @@ type WebtorrentPluginOptions = {
54} 59}
55 60
56type P2PMediaLoaderPluginOptions = { 61type P2PMediaLoaderPluginOptions = {
62 redundancyBaseUrls: string[]
57 type: string 63 type: string
58 src: string 64 src: string
59} 65}
@@ -91,6 +97,13 @@ type AutoResolutionUpdateData = {
91} 97}
92 98
93type PlayerNetworkInfo = { 99type PlayerNetworkInfo = {
100 http: {
101 downloadSpeed: number
102 uploadSpeed: number
103 downloaded: number
104 uploaded: number
105 }
106
94 p2p: { 107 p2p: {
95 downloadSpeed: number 108 downloadSpeed: number
96 uploadSpeed: number 109 uploadSpeed: number
diff --git a/client/src/assets/player/utils.ts b/client/src/assets/player/utils.ts
index 8b9f34b99..8d87567c2 100644
--- a/client/src/assets/player/utils.ts
+++ b/client/src/assets/player/utils.ts
@@ -112,9 +112,23 @@ function videoFileMinByResolution (files: VideoFile[]) {
112 return min 112 return min
113} 113}
114 114
115function getRtcConfig () {
116 return {
117 iceServers: [
118 {
119 urls: 'stun:stun.stunprotocol.org'
120 },
121 {
122 urls: 'stun:stun.framasoft.org'
123 }
124 ]
125 }
126}
127
115// --------------------------------------------------------------------------- 128// ---------------------------------------------------------------------------
116 129
117export { 130export {
131 getRtcConfig,
118 toTitleCase, 132 toTitleCase,
119 timeToInt, 133 timeToInt,
120 buildVideoLink, 134 buildVideoLink,
diff --git a/client/src/assets/player/videojs-components/p2p-info-button.ts b/client/src/assets/player/videojs-components/p2p-info-button.ts
index 2fc4c4562..6424787b2 100644
--- a/client/src/assets/player/videojs-components/p2p-info-button.ts
+++ b/client/src/assets/player/videojs-components/p2p-info-button.ts
@@ -75,11 +75,12 @@ class P2pInfoButton extends Button {
75 } 75 }
76 76
77 const p2pStats = data.p2p 77 const p2pStats = data.p2p
78 const httpStats = data.http
78 79
79 const downloadSpeed = bytes(p2pStats.downloadSpeed) 80 const downloadSpeed = bytes(p2pStats.downloadSpeed + httpStats.downloadSpeed)
80 const uploadSpeed = bytes(p2pStats.uploadSpeed) 81 const uploadSpeed = bytes(p2pStats.uploadSpeed + httpStats.uploadSpeed)
81 const totalDownloaded = bytes(p2pStats.downloaded) 82 const totalDownloaded = bytes(p2pStats.downloaded + httpStats.downloaded)
82 const totalUploaded = bytes(p2pStats.uploaded) 83 const totalUploaded = bytes(p2pStats.uploaded + httpStats.uploaded)
83 const numPeers = p2pStats.numPeers 84 const numPeers = p2pStats.numPeers
84 85
85 subDivWebtorrent.title = this.player_.localize('Total downloaded: ') + totalDownloaded.join(' ') + '\n' + 86 subDivWebtorrent.title = this.player_.localize('Total downloaded: ') + totalDownloaded.join(' ') + '\n' +
@@ -92,7 +93,7 @@ class P2pInfoButton extends Button {
92 uploadSpeedUnit.textContent = ' ' + uploadSpeed[ 1 ] 93 uploadSpeedUnit.textContent = ' ' + uploadSpeed[ 1 ]
93 94
94 peersNumber.textContent = numPeers 95 peersNumber.textContent = numPeers
95 peersText.textContent = ' ' + this.player_.localize('peers') 96 peersText.textContent = ' ' + (numPeers > 1 ? this.player_.localize('peers') : this.player_.localize('peer'))
96 97
97 subDivHttp.className = 'vjs-peertube-hidden' 98 subDivHttp.className = 'vjs-peertube-hidden'
98 subDivWebtorrent.className = 'vjs-peertube-displayed' 99 subDivWebtorrent.className = 'vjs-peertube-displayed'
diff --git a/client/src/assets/player/webtorrent-plugin.ts b/client/src/assets/player/webtorrent/webtorrent-plugin.ts
index 47f169e24..c69bf31fa 100644
--- a/client/src/assets/player/webtorrent-plugin.ts
+++ b/client/src/assets/player/webtorrent/webtorrent-plugin.ts
@@ -3,18 +3,18 @@
3import * as videojs from 'video.js' 3import * as videojs from 'video.js'
4 4
5import * as WebTorrent from 'webtorrent' 5import * as WebTorrent from 'webtorrent'
6import { VideoFile } from '../../../../shared/models/videos/video.model' 6import { VideoFile } from '../../../../../shared/models/videos/video.model'
7import { renderVideo } from './webtorrent/video-renderer' 7import { renderVideo } from './video-renderer'
8import { LoadedQualityData, PlayerNetworkInfo, VideoJSComponentInterface, WebtorrentPluginOptions } from './peertube-videojs-typings' 8import { LoadedQualityData, PlayerNetworkInfo, VideoJSComponentInterface, WebtorrentPluginOptions } from '../peertube-videojs-typings'
9import { videoFileMaxByResolution, videoFileMinByResolution } from './utils' 9import { getRtcConfig, videoFileMaxByResolution, videoFileMinByResolution } from '../utils'
10import { PeertubeChunkStore } from './webtorrent/peertube-chunk-store' 10import { PeertubeChunkStore } from './peertube-chunk-store'
11import { 11import {
12 getAverageBandwidthInStore, 12 getAverageBandwidthInStore,
13 getStoredMute, 13 getStoredMute,
14 getStoredVolume, 14 getStoredVolume,
15 getStoredWebTorrentEnabled, 15 getStoredWebTorrentEnabled,
16 saveAverageBandwidth 16 saveAverageBandwidth
17} from './peertube-player-local-storage' 17} from '../peertube-player-local-storage'
18 18
19const CacheChunkStore = require('cache-chunk-store') 19const CacheChunkStore = require('cache-chunk-store')
20 20
@@ -44,16 +44,7 @@ class WebTorrentPlugin extends Plugin {
44 44
45 private readonly webtorrent = new WebTorrent({ 45 private readonly webtorrent = new WebTorrent({
46 tracker: { 46 tracker: {
47 rtcConfig: { 47 rtcConfig: getRtcConfig()
48 iceServers: [
49 {
50 urls: 'stun:stun.stunprotocol.org'
51 },
52 {
53 urls: 'stun:stun.framasoft.org'
54 }
55 ]
56 }
57 }, 48 },
58 dht: false 49 dht: false
59 }) 50 })
@@ -472,6 +463,12 @@ class WebTorrentPlugin extends Plugin {
472 if (this.webtorrent.downloadSpeed !== 0) this.downloadSpeeds.push(this.webtorrent.downloadSpeed) 463 if (this.webtorrent.downloadSpeed !== 0) this.downloadSpeeds.push(this.webtorrent.downloadSpeed)
473 464
474 return this.player.trigger('p2pInfo', { 465 return this.player.trigger('p2pInfo', {
466 http: {
467 downloadSpeed: 0,
468 uploadSpeed: 0,
469 downloaded: 0,
470 uploaded: 0
471 },
475 p2p: { 472 p2p: {
476 downloadSpeed: this.torrent.downloadSpeed, 473 downloadSpeed: this.torrent.downloadSpeed,
477 numPeers: this.torrent.numPeers, 474 numPeers: this.torrent.numPeers,
diff --git a/client/src/standalone/videos/embed.ts b/client/src/standalone/videos/embed.ts
index 6dd9a3d76..1e58d42d9 100644
--- a/client/src/standalone/videos/embed.ts
+++ b/client/src/standalone/videos/embed.ts
@@ -23,7 +23,13 @@ import { peertubeTranslate, ResultList, VideoDetails } from '../../../../shared'
23import { PeerTubeResolution } from '../player/definitions' 23import { PeerTubeResolution } from '../player/definitions'
24import { VideoJSCaption } from '../../assets/player/peertube-videojs-typings' 24import { VideoJSCaption } from '../../assets/player/peertube-videojs-typings'
25import { VideoCaption } from '../../../../shared/models/videos/caption/video-caption.model' 25import { VideoCaption } from '../../../../shared/models/videos/caption/video-caption.model'
26import { PeertubePlayerManager, PeertubePlayerManagerOptions, PlayerMode } from '../../assets/player/peertube-player-manager' 26import {
27 P2PMediaLoaderOptions,
28 PeertubePlayerManager,
29 PeertubePlayerManagerOptions,
30 PlayerMode
31} from '../../assets/player/peertube-player-manager'
32import { VideoStreamingPlaylistType } from '../../../../shared/models/videos/video-streaming-playlist.type'
27 33
28/** 34/**
29 * Embed API exposes control of the embed player to the outside world via 35 * Embed API exposes control of the embed player to the outside world via
@@ -319,13 +325,16 @@ class PeerTubeEmbed {
319 } 325 }
320 326
321 if (this.mode === 'p2p-media-loader') { 327 if (this.mode === 'p2p-media-loader') {
328 const hlsPlaylist = videoInfo.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
329
322 Object.assign(options, { 330 Object.assign(options, {
323 p2pMediaLoader: { 331 p2pMediaLoader: {
324 // playlistUrl: 'https://akamai-axtest.akamaized.net/routes/lapd-v1-acceptance/www_c4/Manifest.m3u8' 332 playlistUrl: hlsPlaylist.playlistUrl,
325 // playlistUrl: 'https://d2zihajmogu5jn.cloudfront.net/bipbop-advanced/bipbop_16x9_variant.m3u8' 333 segmentsSha256Url: hlsPlaylist.segmentsSha256Url,
326 // trackerAnnounce: [ window.location.origin.replace(/^http/, 'ws') + '/tracker/socket' ], 334 redundancyBaseUrls: hlsPlaylist.redundancies.map(r => r.baseUrl),
327 playlistUrl: 'https://cdn.theoplayer.com/video/elephants-dream/playlist.m3u8' 335 trackerAnnounce: videoInfo.trackerUrls,
328 } 336 videoFiles: videoInfo.files
337 } as P2PMediaLoaderOptions
329 }) 338 })
330 } else { 339 } else {
331 Object.assign(options, { 340 Object.assign(options, {
diff --git a/client/yarn.lock b/client/yarn.lock
index ced35688f..06352908e 100644
--- a/client/yarn.lock
+++ b/client/yarn.lock
@@ -2641,6 +2641,13 @@ debug@^3.1.0, debug@^3.2.5:
2641 dependencies: 2641 dependencies:
2642 ms "^2.1.1" 2642 ms "^2.1.1"
2643 2643
2644debug@^4.1.1:
2645 version "4.1.1"
2646 resolved "https://registry.yarnpkg.com/debug/-/debug-4.1.1.tgz#3b72260255109c6b589cee050f1d516139664791"
2647 integrity sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==
2648 dependencies:
2649 ms "^2.1.1"
2650
2644decamelize@^1.1.1, decamelize@^1.1.2, decamelize@^1.2.0: 2651decamelize@^1.1.1, decamelize@^1.1.2, decamelize@^1.2.0:
2645 version "1.2.0" 2652 version "1.2.0"
2646 resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" 2653 resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
@@ -6131,7 +6138,7 @@ m3u8-parser@4.2.0:
6131 resolved "https://registry.yarnpkg.com/m3u8-parser/-/m3u8-parser-4.2.0.tgz#c8e0785fd17f741f4408b49466889274a9e36447" 6138 resolved "https://registry.yarnpkg.com/m3u8-parser/-/m3u8-parser-4.2.0.tgz#c8e0785fd17f741f4408b49466889274a9e36447"
6132 integrity sha512-LVHw0U6IPJjwk9i9f7Xe26NqaUHTNlIt4SSWoEfYFROeVKHN6MIjOhbRheI3dg8Jbq5WCuMFQ0QU3EgZpmzFPg== 6139 integrity sha512-LVHw0U6IPJjwk9i9f7Xe26NqaUHTNlIt4SSWoEfYFROeVKHN6MIjOhbRheI3dg8Jbq5WCuMFQ0QU3EgZpmzFPg==
6133 6140
6134m3u8-parser@^4.2.0: 6141m3u8-parser@^4.3.0:
6135 version "4.3.0" 6142 version "4.3.0"
6136 resolved "https://registry.yarnpkg.com/m3u8-parser/-/m3u8-parser-4.3.0.tgz#4b4e988f87b6d8b2401d209a1d17798285a9da04" 6143 resolved "https://registry.yarnpkg.com/m3u8-parser/-/m3u8-parser-4.3.0.tgz#4b4e988f87b6d8b2401d209a1d17798285a9da04"
6137 integrity sha512-bVbjuBMoVIgFL1vpXVIxjeaoB5TPDJRb0m5qiTdM738SGqv/LAmsnVVPlKjM4fulm/rr1XZsKM+owHm+zvqxYA== 6144 integrity sha512-bVbjuBMoVIgFL1vpXVIxjeaoB5TPDJRb0m5qiTdM738SGqv/LAmsnVVPlKjM4fulm/rr1XZsKM+owHm+zvqxYA==
@@ -7244,25 +7251,25 @@ p-try@^2.0.0:
7244 resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.0.0.tgz#85080bb87c64688fa47996fe8f7dfbe8211760b1" 7251 resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.0.0.tgz#85080bb87c64688fa47996fe8f7dfbe8211760b1"
7245 integrity sha512-hMp0onDKIajHfIkdRk3P4CdCmErkYAxxDtP3Wx/4nZ3aGlau2VKh3mZpcuFkH27WQkL/3WBCPOktzA9ZOAnMQQ== 7252 integrity sha512-hMp0onDKIajHfIkdRk3P4CdCmErkYAxxDtP3Wx/4nZ3aGlau2VKh3mZpcuFkH27WQkL/3WBCPOktzA9ZOAnMQQ==
7246 7253
7247p2p-media-loader-core@^0.3.0: 7254p2p-media-loader-core@^0.4.0:
7248 version "0.3.0" 7255 version "0.4.0"
7249 resolved "https://registry.yarnpkg.com/p2p-media-loader-core/-/p2p-media-loader-core-0.3.0.tgz#75687d7d7bee835d5c6c2f17d346add2dbe43b83" 7256 resolved "https://registry.yarnpkg.com/p2p-media-loader-core/-/p2p-media-loader-core-0.4.0.tgz#767d56785545bc9c0d8c1a04eb7b67a33e40d0c8"
7250 integrity sha512-WKB9ONdA0kDQHXr6nixIL8t0UZuTD9Pqi/BIuaTiPUGDwYXUS/Mf5YynLAUupniLkIaDYD7/jmSLWqpZUDsAyw== 7257 integrity sha512-llcFqEDs19o916g2OSIPHPjZweO5caHUm/7P18Qu+qb3swYQYSPNwMLoHnpXROHiH5I+00K8w5enz31oUwiCgA==
7251 dependencies: 7258 dependencies:
7252 bittorrent-tracker "^9.10.1" 7259 bittorrent-tracker "^9.10.1"
7253 debug "^4.1.0" 7260 debug "^4.1.1"
7254 events "^3.0.0" 7261 events "^3.0.0"
7255 get-browser-rtc "^1.0.2" 7262 get-browser-rtc "^1.0.2"
7256 sha.js "^2.4.11" 7263 sha.js "^2.4.11"
7257 7264
7258p2p-media-loader-hlsjs@^0.3.0: 7265p2p-media-loader-hlsjs@^0.4.0:
7259 version "0.3.0" 7266 version "0.4.0"
7260 resolved "https://registry.yarnpkg.com/p2p-media-loader-hlsjs/-/p2p-media-loader-hlsjs-0.3.0.tgz#4ee15d4d1a23aa0322a5be2bc6c329b6c913028d" 7267 resolved "https://registry.yarnpkg.com/p2p-media-loader-hlsjs/-/p2p-media-loader-hlsjs-0.4.0.tgz#1b90c88580503d4c3d8017c813abe41803b613ed"
7261 integrity sha512-U7PzMG5X7CVQ15OtMPRQjW68Msu0fuw8Pp0PRznX5uK0p26tSYMT/ZYCNeYCoDg3wGgJHM+327ed3M7TRJ4lcw== 7268 integrity sha512-IWRs/aGasKD//+dtQkYWAjD/cQx3LMaLkMn0EzLhLpeBj4SLNjlbwOPlbx36M4i39X04Y3WZe9YUeIciId3G5Q==
7262 dependencies: 7269 dependencies:
7263 events "^3.0.0" 7270 events "^3.0.0"
7264 m3u8-parser "^4.2.0" 7271 m3u8-parser "^4.3.0"
7265 p2p-media-loader-core "^0.3.0" 7272 p2p-media-loader-core "^0.4.0"
7266 7273
7267package-json-versionify@^1.0.2: 7274package-json-versionify@^1.0.2:
7268 version "1.0.4" 7275 version "1.0.4"
diff --git a/config/default.yaml b/config/default.yaml
index e16b8c352..ad0e6084b 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -48,6 +48,7 @@ storage:
48 tmp: 'storage/tmp/' # Used to download data (imports etc), store uploaded files before processing... 48 tmp: 'storage/tmp/' # Used to download data (imports etc), store uploaded files before processing...
49 avatars: 'storage/avatars/' 49 avatars: 'storage/avatars/'
50 videos: 'storage/videos/' 50 videos: 'storage/videos/'
51 playlists: 'storage/playlists/'
51 redundancy: 'storage/redundancy/' 52 redundancy: 'storage/redundancy/'
52 logs: 'storage/logs/' 53 logs: 'storage/logs/'
53 previews: 'storage/previews/' 54 previews: 'storage/previews/'
@@ -138,6 +139,14 @@ transcoding:
138 480p: false 139 480p: false
139 720p: false 140 720p: false
140 1080p: false 141 1080p: false
142 # /!\ EXPERIMENTAL /!\
143 # Generate HLS playlist/segments. Better playback than with WebTorrent:
144 # * Resolution change is smoother
145 # * Faster playback in particular with long videos
146 # * More stable playback (less bugs/infinite loading)
147 # /!\ Multiply videos storage by two /!\
148 hls:
149 enabled: false
141 150
142import: 151import:
143 # Add ability for your users to import remote videos (from YouTube, torrent...) 152 # Add ability for your users to import remote videos (from YouTube, torrent...)
diff --git a/config/production.yaml.example b/config/production.yaml.example
index 661eac0d5..98734bab6 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -49,6 +49,7 @@ storage:
49 tmp: '/var/www/peertube/storage/tmp/' # Used to download data (imports etc), store uploaded files before processing... 49 tmp: '/var/www/peertube/storage/tmp/' # Used to download data (imports etc), store uploaded files before processing...
50 avatars: '/var/www/peertube/storage/avatars/' 50 avatars: '/var/www/peertube/storage/avatars/'
51 videos: '/var/www/peertube/storage/videos/' 51 videos: '/var/www/peertube/storage/videos/'
52 playlists: '/var/www/peertube/storage/playlists/'
52 redundancy: '/var/www/peertube/storage/videos/' 53 redundancy: '/var/www/peertube/storage/videos/'
53 logs: '/var/www/peertube/storage/logs/' 54 logs: '/var/www/peertube/storage/logs/'
54 previews: '/var/www/peertube/storage/previews/' 55 previews: '/var/www/peertube/storage/previews/'
@@ -151,6 +152,14 @@ transcoding:
151 480p: false 152 480p: false
152 720p: false 153 720p: false
153 1080p: false 154 1080p: false
155 # /!\ EXPERIMENTAL /!\
156 # Generate HLS playlist/segments. Better playback than with WebTorrent:
157 # * Resolution change is smoother
158 # * Faster playback in particular with long videos
159 # * More stable playback (less bugs/infinite loading)
160 # /!\ Multiply videos storage by two /!\
161 hls:
162 enabled: false
154 163
155import: 164import:
156 # Add ability for your users to import remote videos (from YouTube, torrent...) 165 # Add ability for your users to import remote videos (from YouTube, torrent...)
diff --git a/config/test-1.yaml b/config/test-1.yaml
index 8f4f66d2a..fb69818f3 100644
--- a/config/test-1.yaml
+++ b/config/test-1.yaml
@@ -13,6 +13,7 @@ storage:
13 tmp: 'test1/tmp/' 13 tmp: 'test1/tmp/'
14 avatars: 'test1/avatars/' 14 avatars: 'test1/avatars/'
15 videos: 'test1/videos/' 15 videos: 'test1/videos/'
16 playlists: 'test1/playlists/'
16 redundancy: 'test1/redundancy/' 17 redundancy: 'test1/redundancy/'
17 logs: 'test1/logs/' 18 logs: 'test1/logs/'
18 previews: 'test1/previews/' 19 previews: 'test1/previews/'
diff --git a/config/test-2.yaml b/config/test-2.yaml
index b6d319394..5caddaaa8 100644
--- a/config/test-2.yaml
+++ b/config/test-2.yaml
@@ -13,6 +13,7 @@ storage:
13 tmp: 'test2/tmp/' 13 tmp: 'test2/tmp/'
14 avatars: 'test2/avatars/' 14 avatars: 'test2/avatars/'
15 videos: 'test2/videos/' 15 videos: 'test2/videos/'
16 playlists: 'test2/playlists/'
16 redundancy: 'test2/redundancy/' 17 redundancy: 'test2/redundancy/'
17 logs: 'test2/logs/' 18 logs: 'test2/logs/'
18 previews: 'test2/previews/' 19 previews: 'test2/previews/'
diff --git a/config/test-3.yaml b/config/test-3.yaml
index 934401eb0..fac7ebee1 100644
--- a/config/test-3.yaml
+++ b/config/test-3.yaml
@@ -13,6 +13,7 @@ storage:
13 tmp: 'test3/tmp/' 13 tmp: 'test3/tmp/'
14 avatars: 'test3/avatars/' 14 avatars: 'test3/avatars/'
15 videos: 'test3/videos/' 15 videos: 'test3/videos/'
16 playlists: 'test3/playlists/'
16 redundancy: 'test3/redundancy/' 17 redundancy: 'test3/redundancy/'
17 logs: 'test3/logs/' 18 logs: 'test3/logs/'
18 previews: 'test3/previews/' 19 previews: 'test3/previews/'
diff --git a/config/test-4.yaml b/config/test-4.yaml
index ee99b250b..33033773a 100644
--- a/config/test-4.yaml
+++ b/config/test-4.yaml
@@ -13,6 +13,7 @@ storage:
13 tmp: 'test4/tmp/' 13 tmp: 'test4/tmp/'
14 avatars: 'test4/avatars/' 14 avatars: 'test4/avatars/'
15 videos: 'test4/videos/' 15 videos: 'test4/videos/'
16 playlists: 'test4/playlists/'
16 redundancy: 'test4/redundancy/' 17 redundancy: 'test4/redundancy/'
17 logs: 'test4/logs/' 18 logs: 'test4/logs/'
18 previews: 'test4/previews/' 19 previews: 'test4/previews/'
diff --git a/config/test-5.yaml b/config/test-5.yaml
index e2662bdd9..d365b6f2b 100644
--- a/config/test-5.yaml
+++ b/config/test-5.yaml
@@ -13,6 +13,7 @@ storage:
13 tmp: 'test5/tmp/' 13 tmp: 'test5/tmp/'
14 avatars: 'test5/avatars/' 14 avatars: 'test5/avatars/'
15 videos: 'test5/videos/' 15 videos: 'test5/videos/'
16 playlists: 'test5/playlists/'
16 redundancy: 'test5/redundancy/' 17 redundancy: 'test5/redundancy/'
17 logs: 'test5/logs/' 18 logs: 'test5/logs/'
18 previews: 'test5/previews/' 19 previews: 'test5/previews/'
diff --git a/config/test-6.yaml b/config/test-6.yaml
index ad39c6a9f..44541c003 100644
--- a/config/test-6.yaml
+++ b/config/test-6.yaml
@@ -13,6 +13,7 @@ storage:
13 tmp: 'test6/tmp/' 13 tmp: 'test6/tmp/'
14 avatars: 'test6/avatars/' 14 avatars: 'test6/avatars/'
15 videos: 'test6/videos/' 15 videos: 'test6/videos/'
16 playlists: 'test6/playlists/'
16 redundancy: 'test6/redundancy/' 17 redundancy: 'test6/redundancy/'
17 logs: 'test6/logs/' 18 logs: 'test6/logs/'
18 previews: 'test6/previews/' 19 previews: 'test6/previews/'
diff --git a/config/test.yaml b/config/test.yaml
index aba5dd73c..682530840 100644
--- a/config/test.yaml
+++ b/config/test.yaml
@@ -62,6 +62,8 @@ transcoding:
62 480p: true 62 480p: true
63 720p: true 63 720p: true
64 1080p: true 64 1080p: true
65 hls:
66 enabled: true
65 67
66import: 68import:
67 videos: 69 videos:
diff --git a/package.json b/package.json
index 0cf39c7ee..c8c9e64ae 100644
--- a/package.json
+++ b/package.json
@@ -117,6 +117,7 @@
117 "fluent-ffmpeg": "^2.1.0", 117 "fluent-ffmpeg": "^2.1.0",
118 "fs-extra": "^7.0.0", 118 "fs-extra": "^7.0.0",
119 "helmet": "^3.12.1", 119 "helmet": "^3.12.1",
120 "hlsdownloader": "https://github.com/Chocobozzz/hlsdownloader#build",
120 "http-signature": "^1.2.0", 121 "http-signature": "^1.2.0",
121 "ip-anonymize": "^0.0.6", 122 "ip-anonymize": "^0.0.6",
122 "ipaddr.js": "1.8.1", 123 "ipaddr.js": "1.8.1",
diff --git a/scripts/i18n/create-custom-files.ts b/scripts/i18n/create-custom-files.ts
index ab28f94c8..664207e1c 100755
--- a/scripts/i18n/create-custom-files.ts
+++ b/scripts/i18n/create-custom-files.ts
@@ -23,12 +23,15 @@ const playerKeys = {
23 'Speed': 'Speed', 23 'Speed': 'Speed',
24 'Subtitles/CC': 'Subtitles/CC', 24 'Subtitles/CC': 'Subtitles/CC',
25 'peers': 'peers', 25 'peers': 'peers',
26 'peer': 'peer',
26 'Go to the video page': 'Go to the video page', 27 'Go to the video page': 'Go to the video page',
27 'Settings': 'Settings', 28 'Settings': 'Settings',
28 'Uses P2P, others may know you are watching this video.': 'Uses P2P, others may know you are watching this video.', 29 'Uses P2P, others may know you are watching this video.': 'Uses P2P, others may know you are watching this video.',
29 'Copy the video URL': 'Copy the video URL', 30 'Copy the video URL': 'Copy the video URL',
30 'Copy the video URL at the current time': 'Copy the video URL at the current time', 31 'Copy the video URL at the current time': 'Copy the video URL at the current time',
31 'Copy embed code': 'Copy embed code' 32 'Copy embed code': 'Copy embed code',
33 'Total downloaded: ': 'Total downloaded: ',
34 'Total uploaded: ': 'Total uploaded: '
32} 35}
33const playerTranslations = { 36const playerTranslations = {
34 target: join(__dirname, '../../../client/src/locale/source/player_en_US.xml'), 37 target: join(__dirname, '../../../client/src/locale/source/player_en_US.xml'),
diff --git a/scripts/update-host.ts b/scripts/update-host.ts
index 422a3c9a7..64eba867a 100755
--- a/scripts/update-host.ts
+++ b/scripts/update-host.ts
@@ -13,6 +13,7 @@ import { VideoCommentModel } from '../server/models/video/video-comment'
13import { getServerActor } from '../server/helpers/utils' 13import { getServerActor } from '../server/helpers/utils'
14import { AccountModel } from '../server/models/account/account' 14import { AccountModel } from '../server/models/account/account'
15import { VideoChannelModel } from '../server/models/video/video-channel' 15import { VideoChannelModel } from '../server/models/video/video-channel'
16import { VideoStreamingPlaylistModel } from '../server/models/video/video-streaming-playlist'
16 17
17run() 18run()
18 .then(() => process.exit(0)) 19 .then(() => process.exit(0))
@@ -109,11 +110,9 @@ async function run () {
109 110
110 console.log('Updating video and torrent files.') 111 console.log('Updating video and torrent files.')
111 112
112 const videos = await VideoModel.list() 113 const videos = await VideoModel.listLocal()
113 for (const video of videos) { 114 for (const video of videos) {
114 if (video.isOwned() === false) continue 115 console.log('Updating video ' + video.uuid)
115
116 console.log('Updated video ' + video.uuid)
117 116
118 video.url = getVideoActivityPubUrl(video) 117 video.url = getVideoActivityPubUrl(video)
119 await video.save() 118 await video.save()
@@ -122,5 +121,12 @@ async function run () {
122 console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid) 121 console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid)
123 await video.createTorrentAndSetInfoHash(file) 122 await video.createTorrentAndSetInfoHash(file)
124 } 123 }
124
125 for (const playlist of video.VideoStreamingPlaylists) {
126 playlist.playlistUrl = CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
127 playlist.segmentsSha256Url = CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid)
128
129 await playlist.save()
130 }
125 } 131 }
126} 132}
diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts
index 1a4e28dc8..32a83aa5f 100644
--- a/server/controllers/activitypub/client.ts
+++ b/server/controllers/activitypub/client.ts
@@ -37,7 +37,7 @@ import {
37 getVideoSharesActivityPubUrl 37 getVideoSharesActivityPubUrl
38} from '../../lib/activitypub' 38} from '../../lib/activitypub'
39import { VideoCaptionModel } from '../../models/video/video-caption' 39import { VideoCaptionModel } from '../../models/video/video-caption'
40import { videoRedundancyGetValidator } from '../../middlewares/validators/redundancy' 40import { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy'
41import { getServerActor } from '../../helpers/utils' 41import { getServerActor } from '../../helpers/utils'
42import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 42import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
43 43
@@ -66,11 +66,11 @@ activityPubClientRouter.get('/accounts?/:name/dislikes/:videoId',
66 66
67activityPubClientRouter.get('/videos/watch/:id', 67activityPubClientRouter.get('/videos/watch/:id',
68 executeIfActivityPub(asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS))), 68 executeIfActivityPub(asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS))),
69 executeIfActivityPub(asyncMiddleware(videosGetValidator)), 69 executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video-with-rights'))),
70 executeIfActivityPub(asyncMiddleware(videoController)) 70 executeIfActivityPub(asyncMiddleware(videoController))
71) 71)
72activityPubClientRouter.get('/videos/watch/:id/activity', 72activityPubClientRouter.get('/videos/watch/:id/activity',
73 executeIfActivityPub(asyncMiddleware(videosGetValidator)), 73 executeIfActivityPub(asyncMiddleware(videosCustomGetValidator('only-video-with-rights'))),
74 executeIfActivityPub(asyncMiddleware(videoController)) 74 executeIfActivityPub(asyncMiddleware(videoController))
75) 75)
76activityPubClientRouter.get('/videos/watch/:id/announces', 76activityPubClientRouter.get('/videos/watch/:id/announces',
@@ -116,7 +116,11 @@ activityPubClientRouter.get('/video-channels/:name/following',
116) 116)
117 117
118activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?', 118activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?',
119 executeIfActivityPub(asyncMiddleware(videoRedundancyGetValidator)), 119 executeIfActivityPub(asyncMiddleware(videoFileRedundancyGetValidator)),
120 executeIfActivityPub(asyncMiddleware(videoRedundancyController))
121)
122activityPubClientRouter.get('/redundancy/video-playlists/:streamingPlaylistType/:videoId',
123 executeIfActivityPub(asyncMiddleware(videoPlaylistRedundancyGetValidator)),
120 executeIfActivityPub(asyncMiddleware(videoRedundancyController)) 124 executeIfActivityPub(asyncMiddleware(videoRedundancyController))
121) 125)
122 126
@@ -163,7 +167,8 @@ function getAccountVideoRate (rateType: VideoRateType) {
163} 167}
164 168
165async function videoController (req: express.Request, res: express.Response) { 169async function videoController (req: express.Request, res: express.Response) {
166 const video: VideoModel = res.locals.video 170 // We need more attributes
171 const video: VideoModel = await VideoModel.loadForGetAPI(res.locals.video.id)
167 172
168 if (video.url.startsWith(CONFIG.WEBSERVER.URL) === false) return res.redirect(video.url) 173 if (video.url.startsWith(CONFIG.WEBSERVER.URL) === false) return res.redirect(video.url)
169 174
diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts
index 255026f46..1f3341bc0 100644
--- a/server/controllers/api/config.ts
+++ b/server/controllers/api/config.ts
@@ -1,5 +1,5 @@
1import * as express from 'express' 1import * as express from 'express'
2import { omit, snakeCase } from 'lodash' 2import { snakeCase } from 'lodash'
3import { ServerConfig, UserRight } from '../../../shared' 3import { ServerConfig, UserRight } from '../../../shared'
4import { About } from '../../../shared/models/server/about.model' 4import { About } from '../../../shared/models/server/about.model'
5import { CustomConfig } from '../../../shared/models/server/custom-config.model' 5import { CustomConfig } from '../../../shared/models/server/custom-config.model'
@@ -78,6 +78,9 @@ async function getConfig (req: express.Request, res: express.Response) {
78 requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION 78 requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION
79 }, 79 },
80 transcoding: { 80 transcoding: {
81 hls: {
82 enabled: CONFIG.TRANSCODING.HLS.ENABLED
83 },
81 enabledResolutions 84 enabledResolutions
82 }, 85 },
83 import: { 86 import: {
@@ -246,6 +249,9 @@ function customConfig (): CustomConfig {
246 '480p': CONFIG.TRANSCODING.RESOLUTIONS[ '480p' ], 249 '480p': CONFIG.TRANSCODING.RESOLUTIONS[ '480p' ],
247 '720p': CONFIG.TRANSCODING.RESOLUTIONS[ '720p' ], 250 '720p': CONFIG.TRANSCODING.RESOLUTIONS[ '720p' ],
248 '1080p': CONFIG.TRANSCODING.RESOLUTIONS[ '1080p' ] 251 '1080p': CONFIG.TRANSCODING.RESOLUTIONS[ '1080p' ]
252 },
253 hls: {
254 enabled: CONFIG.TRANSCODING.HLS.ENABLED
249 } 255 }
250 }, 256 },
251 import: { 257 import: {
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index 2b2dfa7ca..e04fc8186 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -37,6 +37,7 @@ import {
37 setDefaultPagination, 37 setDefaultPagination,
38 setDefaultSort, 38 setDefaultSort,
39 videosAddValidator, 39 videosAddValidator,
40 videosCustomGetValidator,
40 videosGetValidator, 41 videosGetValidator,
41 videosRemoveValidator, 42 videosRemoveValidator,
42 videosSortValidator, 43 videosSortValidator,
@@ -123,9 +124,9 @@ videosRouter.get('/:id/description',
123) 124)
124videosRouter.get('/:id', 125videosRouter.get('/:id',
125 optionalAuthenticate, 126 optionalAuthenticate,
126 asyncMiddleware(videosGetValidator), 127 asyncMiddleware(videosCustomGetValidator('only-video-with-rights')),
127 asyncMiddleware(checkVideoFollowConstraints), 128 asyncMiddleware(checkVideoFollowConstraints),
128 getVideo 129 asyncMiddleware(getVideo)
129) 130)
130videosRouter.post('/:id/views', 131videosRouter.post('/:id/views',
131 asyncMiddleware(videosGetValidator), 132 asyncMiddleware(videosGetValidator),
@@ -395,15 +396,17 @@ async function updateVideo (req: express.Request, res: express.Response) {
395 return res.type('json').status(204).end() 396 return res.type('json').status(204).end()
396} 397}
397 398
398function getVideo (req: express.Request, res: express.Response) { 399async function getVideo (req: express.Request, res: express.Response) {
399 const videoInstance = res.locals.video 400 // We need more attributes
401 const userId: number = res.locals.oauth ? res.locals.oauth.token.User.id : null
402 const video: VideoModel = await VideoModel.loadForGetAPI(res.locals.video.id, undefined, userId)
400 403
401 if (videoInstance.isOutdated()) { 404 if (video.isOutdated()) {
402 JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoInstance.url } }) 405 JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: video.url } })
403 .catch(err => logger.error('Cannot create AP refresher job for video %s.', videoInstance.url, { err })) 406 .catch(err => logger.error('Cannot create AP refresher job for video %s.', video.url, { err }))
404 } 407 }
405 408
406 return res.json(videoInstance.toFormattedDetailsJSON()) 409 return res.json(video.toFormattedDetailsJSON())
407} 410}
408 411
409async function viewVideo (req: express.Request, res: express.Response) { 412async function viewVideo (req: express.Request, res: express.Response) {
diff --git a/server/controllers/static.ts b/server/controllers/static.ts
index 4fd58f70c..b21f9da00 100644
--- a/server/controllers/static.ts
+++ b/server/controllers/static.ts
@@ -1,6 +1,6 @@
1import * as cors from 'cors' 1import * as cors from 'cors'
2import * as express from 'express' 2import * as express from 'express'
3import { CONFIG, ROUTE_CACHE_LIFETIME, STATIC_DOWNLOAD_PATHS, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers' 3import { CONFIG, HLS_PLAYLIST_DIRECTORY, ROUTE_CACHE_LIFETIME, STATIC_DOWNLOAD_PATHS, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers'
4import { VideosPreviewCache } from '../lib/cache' 4import { VideosPreviewCache } from '../lib/cache'
5import { cacheRoute } from '../middlewares/cache' 5import { cacheRoute } from '../middlewares/cache'
6import { asyncMiddleware, videosGetValidator } from '../middlewares' 6import { asyncMiddleware, videosGetValidator } from '../middlewares'
@@ -51,6 +51,13 @@ staticRouter.use(
51 asyncMiddleware(downloadVideoFile) 51 asyncMiddleware(downloadVideoFile)
52) 52)
53 53
54// HLS
55staticRouter.use(
56 STATIC_PATHS.PLAYLISTS.HLS,
57 cors(),
58 express.static(HLS_PLAYLIST_DIRECTORY, { fallthrough: false }) // 404 if the file does not exist
59)
60
54// Thumbnails path for express 61// Thumbnails path for express
55const thumbnailsPhysicalPath = CONFIG.STORAGE.THUMBNAILS_DIR 62const thumbnailsPhysicalPath = CONFIG.STORAGE.THUMBNAILS_DIR
56staticRouter.use( 63staticRouter.use(
diff --git a/server/controllers/tracker.ts b/server/controllers/tracker.ts
index 1deb8c402..8b77d9de7 100644
--- a/server/controllers/tracker.ts
+++ b/server/controllers/tracker.ts
@@ -7,6 +7,7 @@ import { Server as WebSocketServer } from 'ws'
7import { CONFIG, TRACKER_RATE_LIMITS } from '../initializers/constants' 7import { CONFIG, TRACKER_RATE_LIMITS } from '../initializers/constants'
8import { VideoFileModel } from '../models/video/video-file' 8import { VideoFileModel } from '../models/video/video-file'
9import { parse } from 'url' 9import { parse } from 'url'
10import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
10 11
11const TrackerServer = bitTorrentTracker.Server 12const TrackerServer = bitTorrentTracker.Server
12 13
@@ -21,7 +22,7 @@ const trackerServer = new TrackerServer({
21 udp: false, 22 udp: false,
22 ws: false, 23 ws: false,
23 dht: false, 24 dht: false,
24 filter: function (infoHash, params, cb) { 25 filter: async function (infoHash, params, cb) {
25 let ip: string 26 let ip: string
26 27
27 if (params.type === 'ws') { 28 if (params.type === 'ws') {
@@ -32,19 +33,25 @@ const trackerServer = new TrackerServer({
32 33
33 const key = ip + '-' + infoHash 34 const key = ip + '-' + infoHash
34 35
35 peersIps[ip] = peersIps[ip] ? peersIps[ip] + 1 : 1 36 peersIps[ ip ] = peersIps[ ip ] ? peersIps[ ip ] + 1 : 1
36 peersIpInfoHash[key] = peersIpInfoHash[key] ? peersIpInfoHash[key] + 1 : 1 37 peersIpInfoHash[ key ] = peersIpInfoHash[ key ] ? peersIpInfoHash[ key ] + 1 : 1
37 38
38 if (peersIpInfoHash[key] > TRACKER_RATE_LIMITS.ANNOUNCES_PER_IP_PER_INFOHASH) { 39 if (peersIpInfoHash[ key ] > TRACKER_RATE_LIMITS.ANNOUNCES_PER_IP_PER_INFOHASH) {
39 return cb(new Error(`Too many requests (${peersIpInfoHash[ key ]} of ip ${ip} for torrent ${infoHash}`)) 40 return cb(new Error(`Too many requests (${peersIpInfoHash[ key ]} of ip ${ip} for torrent ${infoHash}`))
40 } 41 }
41 42
42 VideoFileModel.isInfohashExists(infoHash) 43 try {
43 .then(exists => { 44 const videoFileExists = await VideoFileModel.doesInfohashExist(infoHash)
44 if (exists === false) return cb(new Error(`Unknown infoHash ${infoHash}`)) 45 if (videoFileExists === true) return cb()
45 46
46 return cb() 47 const playlistExists = await VideoStreamingPlaylistModel.doesInfohashExist(infoHash)
47 }) 48 if (playlistExists === true) return cb()
49
50 return cb(new Error(`Unknown infoHash ${infoHash}`))
51 } catch (err) {
52 logger.error('Error in tracker filter.', { err })
53 return cb(err)
54 }
48 } 55 }
49}) 56})
50 57
diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts
index f1430055f..eba552524 100644
--- a/server/helpers/activitypub.ts
+++ b/server/helpers/activitypub.ts
@@ -15,7 +15,7 @@ function activityPubContextify <T> (data: T) {
15 'https://w3id.org/security/v1', 15 'https://w3id.org/security/v1',
16 { 16 {
17 RsaSignature2017: 'https://w3id.org/security#RsaSignature2017', 17 RsaSignature2017: 'https://w3id.org/security#RsaSignature2017',
18 pt: 'https://joinpeertube.org/ns', 18 pt: 'https://joinpeertube.org/ns#',
19 sc: 'http://schema.org#', 19 sc: 'http://schema.org#',
20 Hashtag: 'as:Hashtag', 20 Hashtag: 'as:Hashtag',
21 uuid: 'sc:identifier', 21 uuid: 'sc:identifier',
@@ -32,7 +32,8 @@ function activityPubContextify <T> (data: T) {
32 waitTranscoding: 'sc:Boolean', 32 waitTranscoding: 'sc:Boolean',
33 expires: 'sc:expires', 33 expires: 'sc:expires',
34 support: 'sc:Text', 34 support: 'sc:Text',
35 CacheFile: 'pt:CacheFile' 35 CacheFile: 'pt:CacheFile',
36 Infohash: 'pt:Infohash'
36 }, 37 },
37 { 38 {
38 likes: { 39 likes: {
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index 3fb824e36..f38b82d97 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -193,10 +193,14 @@ function peertubeTruncate (str: string, maxLength: number) {
193 return truncate(str, options) 193 return truncate(str, options)
194} 194}
195 195
196function sha256 (str: string, encoding: HexBase64Latin1Encoding = 'hex') { 196function sha256 (str: string | Buffer, encoding: HexBase64Latin1Encoding = 'hex') {
197 return createHash('sha256').update(str).digest(encoding) 197 return createHash('sha256').update(str).digest(encoding)
198} 198}
199 199
200function sha1 (str: string | Buffer, encoding: HexBase64Latin1Encoding = 'hex') {
201 return createHash('sha1').update(str).digest(encoding)
202}
203
200function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> { 204function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> {
201 return function promisified (): Promise<A> { 205 return function promisified (): Promise<A> {
202 return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { 206 return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
@@ -262,7 +266,9 @@ export {
262 sanitizeHost, 266 sanitizeHost,
263 buildPath, 267 buildPath,
264 peertubeTruncate, 268 peertubeTruncate,
269
265 sha256, 270 sha256,
271 sha1,
266 272
267 promisify0, 273 promisify0,
268 promisify1, 274 promisify1,
diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts
index e2bd0c55e..21d5c53ca 100644
--- a/server/helpers/custom-validators/activitypub/cache-file.ts
+++ b/server/helpers/custom-validators/activitypub/cache-file.ts
@@ -8,9 +8,19 @@ function isCacheFileObjectValid (object: CacheFileObject) {
8 object.type === 'CacheFile' && 8 object.type === 'CacheFile' &&
9 isDateValid(object.expires) && 9 isDateValid(object.expires) &&
10 isActivityPubUrlValid(object.object) && 10 isActivityPubUrlValid(object.object) &&
11 isRemoteVideoUrlValid(object.url) 11 (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url))
12} 12}
13 13
14// ---------------------------------------------------------------------------
15
14export { 16export {
15 isCacheFileObjectValid 17 isCacheFileObjectValid
16} 18}
19
20// ---------------------------------------------------------------------------
21
22function isPlaylistRedundancyUrlValid (url: any) {
23 return url.type === 'Link' &&
24 (url.mediaType || url.mimeType) === 'application/x-mpegURL' &&
25 isActivityPubUrlValid(url.href)
26}
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index 0f34aab21..ad99c2724 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -1,7 +1,7 @@
1import * as validator from 'validator' 1import * as validator from 'validator'
2import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers' 2import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers'
3import { peertubeTruncate } from '../../core-utils' 3import { peertubeTruncate } from '../../core-utils'
4import { exists, isBooleanValid, isDateValid, isUUIDValid } from '../misc' 4import { exists, isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc'
5import { 5import {
6 isVideoDurationValid, 6 isVideoDurationValid,
7 isVideoNameValid, 7 isVideoNameValid,
@@ -12,7 +12,6 @@ import {
12} from '../videos' 12} from '../videos'
13import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc' 13import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc'
14import { VideoState } from '../../../../shared/models/videos' 14import { VideoState } from '../../../../shared/models/videos'
15import { isVideoAbuseReasonValid } from '../video-abuses'
16 15
17function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { 16function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) {
18 return isBaseActivityValid(activity, 'Update') && 17 return isBaseActivityValid(activity, 'Update') &&
@@ -81,6 +80,11 @@ function isRemoteVideoUrlValid (url: any) {
81 ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mediaType || url.mimeType) !== -1 && 80 ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mediaType || url.mimeType) !== -1 &&
82 validator.isLength(url.href, { min: 5 }) && 81 validator.isLength(url.href, { min: 5 }) &&
83 validator.isInt(url.height + '', { min: 0 }) 82 validator.isInt(url.height + '', { min: 0 })
83 ) ||
84 (
85 (url.mediaType || url.mimeType) === 'application/x-mpegURL' &&
86 isActivityPubUrlValid(url.href) &&
87 isArray(url.tag)
84 ) 88 )
85} 89}
86 90
diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts
index b6f0ebe6f..76647fea2 100644
--- a/server/helpers/custom-validators/misc.ts
+++ b/server/helpers/custom-validators/misc.ts
@@ -13,6 +13,10 @@ function isNotEmptyIntArray (value: any) {
13 return Array.isArray(value) && value.every(v => validator.isInt('' + v)) && value.length !== 0 13 return Array.isArray(value) && value.every(v => validator.isInt('' + v)) && value.length !== 0
14} 14}
15 15
16function isArrayOf (value: any, validator: (value: any) => boolean) {
17 return isArray(value) && value.every(v => validator(v))
18}
19
16function isDateValid (value: string) { 20function isDateValid (value: string) {
17 return exists(value) && validator.isISO8601(value) 21 return exists(value) && validator.isISO8601(value)
18} 22}
@@ -82,6 +86,7 @@ function isFileValid (
82 86
83export { 87export {
84 exists, 88 exists,
89 isArrayOf,
85 isNotEmptyIntArray, 90 isNotEmptyIntArray,
86 isArray, 91 isArray,
87 isIdValid, 92 isIdValid,
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts
index 132f4690e..5ad8ed48e 100644
--- a/server/helpers/ffmpeg-utils.ts
+++ b/server/helpers/ffmpeg-utils.ts
@@ -1,5 +1,5 @@
1import * as ffmpeg from 'fluent-ffmpeg' 1import * as ffmpeg from 'fluent-ffmpeg'
2import { join } from 'path' 2import { dirname, join } from 'path'
3import { getTargetBitrate, VideoResolution } from '../../shared/models/videos' 3import { getTargetBitrate, VideoResolution } from '../../shared/models/videos'
4import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants' 4import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants'
5import { processImage } from './image-utils' 5import { processImage } from './image-utils'
@@ -29,12 +29,21 @@ function computeResolutionsToTranscode (videoFileHeight: number) {
29 return resolutionsEnabled 29 return resolutionsEnabled
30} 30}
31 31
32async function getVideoFileResolution (path: string) { 32async function getVideoFileSize (path: string) {
33 const videoStream = await getVideoFileStream(path) 33 const videoStream = await getVideoFileStream(path)
34 34
35 return { 35 return {
36 videoFileResolution: Math.min(videoStream.height, videoStream.width), 36 width: videoStream.width,
37 isPortraitMode: videoStream.height > videoStream.width 37 height: videoStream.height
38 }
39}
40
41async function getVideoFileResolution (path: string) {
42 const size = await getVideoFileSize(path)
43
44 return {
45 videoFileResolution: Math.min(size.height, size.width),
46 isPortraitMode: size.height > size.width
38 } 47 }
39} 48}
40 49
@@ -110,8 +119,10 @@ async function generateImageFromVideoFile (fromPath: string, folder: string, ima
110type TranscodeOptions = { 119type TranscodeOptions = {
111 inputPath: string 120 inputPath: string
112 outputPath: string 121 outputPath: string
113 resolution?: VideoResolution 122 resolution: VideoResolution
114 isPortraitMode?: boolean 123 isPortraitMode?: boolean
124
125 generateHlsPlaylist?: boolean
115} 126}
116 127
117function transcode (options: TranscodeOptions) { 128function transcode (options: TranscodeOptions) {
@@ -150,6 +161,16 @@ function transcode (options: TranscodeOptions) {
150 command = command.withFPS(fps) 161 command = command.withFPS(fps)
151 } 162 }
152 163
164 if (options.generateHlsPlaylist) {
165 const segmentFilename = `${dirname(options.outputPath)}/${options.resolution}_%03d.ts`
166
167 command = command.outputOption('-hls_time 4')
168 .outputOption('-hls_list_size 0')
169 .outputOption('-hls_playlist_type vod')
170 .outputOption('-hls_segment_filename ' + segmentFilename)
171 .outputOption('-f hls')
172 }
173
153 command 174 command
154 .on('error', (err, stdout, stderr) => { 175 .on('error', (err, stdout, stderr) => {
155 logger.error('Error in transcoding job.', { stdout, stderr }) 176 logger.error('Error in transcoding job.', { stdout, stderr })
@@ -166,6 +187,7 @@ function transcode (options: TranscodeOptions) {
166// --------------------------------------------------------------------------- 187// ---------------------------------------------------------------------------
167 188
168export { 189export {
190 getVideoFileSize,
169 getVideoFileResolution, 191 getVideoFileResolution,
170 getDurationFromVideoFile, 192 getDurationFromVideoFile,
171 generateImageFromVideoFile, 193 generateImageFromVideoFile,
diff --git a/server/helpers/video.ts b/server/helpers/video.ts
index 1bd21467d..c90fe06c7 100644
--- a/server/helpers/video.ts
+++ b/server/helpers/video.ts
@@ -1,10 +1,12 @@
1import { VideoModel } from '../models/video/video' 1import { VideoModel } from '../models/video/video'
2 2
3type VideoFetchType = 'all' | 'only-video' | 'id' | 'none' 3type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none'
4 4
5function fetchVideo (id: number | string, fetchType: VideoFetchType, userId?: number) { 5function fetchVideo (id: number | string, fetchType: VideoFetchType, userId?: number) {
6 if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId) 6 if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId)
7 7
8 if (fetchType === 'only-video-with-rights') return VideoModel.loadWithRights(id)
9
8 if (fetchType === 'only-video') return VideoModel.load(id) 10 if (fetchType === 'only-video') return VideoModel.load(id)
9 11
10 if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id) 12 if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id)
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index 7905d9ffa..29fdb263e 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -12,7 +12,7 @@ function checkMissedConfig () {
12 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max', 12 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max',
13 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address', 13 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address',
14 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache', 14 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache',
15 'storage.redundancy', 'storage.tmp', 15 'storage.redundancy', 'storage.tmp', 'storage.playlists',
16 'log.level', 16 'log.level',
17 'user.video_quota', 'user.video_quota_daily', 17 'user.video_quota', 'user.video_quota_daily',
18 'cache.previews.size', 'admin.email', 'contact_form.enabled', 18 'cache.previews.size', 'admin.email', 'contact_form.enabled',
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 6f3ebb9aa..98f8f8694 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -16,7 +16,7 @@ let config: IConfig = require('config')
16 16
17// --------------------------------------------------------------------------- 17// ---------------------------------------------------------------------------
18 18
19const LAST_MIGRATION_VERSION = 325 19const LAST_MIGRATION_VERSION = 330
20 20
21// --------------------------------------------------------------------------- 21// ---------------------------------------------------------------------------
22 22
@@ -192,6 +192,7 @@ const CONFIG = {
192 AVATARS_DIR: buildPath(config.get<string>('storage.avatars')), 192 AVATARS_DIR: buildPath(config.get<string>('storage.avatars')),
193 LOG_DIR: buildPath(config.get<string>('storage.logs')), 193 LOG_DIR: buildPath(config.get<string>('storage.logs')),
194 VIDEOS_DIR: buildPath(config.get<string>('storage.videos')), 194 VIDEOS_DIR: buildPath(config.get<string>('storage.videos')),
195 PLAYLISTS_DIR: buildPath(config.get<string>('storage.playlists')),
195 REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')), 196 REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')),
196 THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')), 197 THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')),
197 PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')), 198 PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')),
@@ -259,6 +260,9 @@ const CONFIG = {
259 get '480p' () { return config.get<boolean>('transcoding.resolutions.480p') }, 260 get '480p' () { return config.get<boolean>('transcoding.resolutions.480p') },
260 get '720p' () { return config.get<boolean>('transcoding.resolutions.720p') }, 261 get '720p' () { return config.get<boolean>('transcoding.resolutions.720p') },
261 get '1080p' () { return config.get<boolean>('transcoding.resolutions.1080p') } 262 get '1080p' () { return config.get<boolean>('transcoding.resolutions.1080p') }
263 },
264 HLS: {
265 get ENABLED () { return config.get<boolean>('transcoding.hls.enabled') }
262 } 266 }
263 }, 267 },
264 IMPORT: { 268 IMPORT: {
@@ -590,6 +594,9 @@ const STATIC_PATHS = {
590 TORRENTS: '/static/torrents/', 594 TORRENTS: '/static/torrents/',
591 WEBSEED: '/static/webseed/', 595 WEBSEED: '/static/webseed/',
592 REDUNDANCY: '/static/redundancy/', 596 REDUNDANCY: '/static/redundancy/',
597 PLAYLISTS: {
598 HLS: '/static/playlists/hls'
599 },
593 AVATARS: '/static/avatars/', 600 AVATARS: '/static/avatars/',
594 VIDEO_CAPTIONS: '/static/video-captions/' 601 VIDEO_CAPTIONS: '/static/video-captions/'
595} 602}
@@ -632,6 +639,9 @@ const CACHE = {
632 } 639 }
633} 640}
634 641
642const HLS_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.PLAYLISTS_DIR, 'hls')
643const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
644
635const MEMOIZE_TTL = { 645const MEMOIZE_TTL = {
636 OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours 646 OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours
637} 647}
@@ -709,6 +719,7 @@ updateWebserverUrls()
709 719
710export { 720export {
711 API_VERSION, 721 API_VERSION,
722 HLS_REDUNDANCY_DIRECTORY,
712 AVATARS_SIZE, 723 AVATARS_SIZE,
713 ACCEPT_HEADERS, 724 ACCEPT_HEADERS,
714 BCRYPT_SALT_SIZE, 725 BCRYPT_SALT_SIZE,
@@ -733,6 +744,7 @@ export {
733 PRIVATE_RSA_KEY_SIZE, 744 PRIVATE_RSA_KEY_SIZE,
734 ROUTE_CACHE_LIFETIME, 745 ROUTE_CACHE_LIFETIME,
735 SORTABLE_COLUMNS, 746 SORTABLE_COLUMNS,
747 HLS_PLAYLIST_DIRECTORY,
736 FEEDS, 748 FEEDS,
737 JOB_TTL, 749 JOB_TTL,
738 NSFW_POLICY_TYPES, 750 NSFW_POLICY_TYPES,
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 84ad2079b..fe296142d 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -33,6 +33,7 @@ import { AccountBlocklistModel } from '../models/account/account-blocklist'
33import { ServerBlocklistModel } from '../models/server/server-blocklist' 33import { ServerBlocklistModel } from '../models/server/server-blocklist'
34import { UserNotificationModel } from '../models/account/user-notification' 34import { UserNotificationModel } from '../models/account/user-notification'
35import { UserNotificationSettingModel } from '../models/account/user-notification-setting' 35import { UserNotificationSettingModel } from '../models/account/user-notification-setting'
36import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
36 37
37require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string 38require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
38 39
@@ -99,7 +100,8 @@ async function initDatabaseModels (silent: boolean) {
99 AccountBlocklistModel, 100 AccountBlocklistModel,
100 ServerBlocklistModel, 101 ServerBlocklistModel,
101 UserNotificationModel, 102 UserNotificationModel,
102 UserNotificationSettingModel 103 UserNotificationSettingModel,
104 VideoStreamingPlaylistModel
103 ]) 105 ])
104 106
105 // Check extensions exist in the database 107 // Check extensions exist in the database
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts
index b9a9da183..2b22e16fe 100644
--- a/server/initializers/installer.ts
+++ b/server/initializers/installer.ts
@@ -6,7 +6,7 @@ import { UserModel } from '../models/account/user'
6import { ApplicationModel } from '../models/application/application' 6import { ApplicationModel } from '../models/application/application'
7import { OAuthClientModel } from '../models/oauth/oauth-client' 7import { OAuthClientModel } from '../models/oauth/oauth-client'
8import { applicationExist, clientsExist, usersExist } from './checker-after-init' 8import { applicationExist, clientsExist, usersExist } from './checker-after-init'
9import { CACHE, CONFIG, LAST_MIGRATION_VERSION } from './constants' 9import { CACHE, CONFIG, HLS_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants'
10import { sequelizeTypescript } from './database' 10import { sequelizeTypescript } from './database'
11import { remove, ensureDir } from 'fs-extra' 11import { remove, ensureDir } from 'fs-extra'
12 12
@@ -73,6 +73,9 @@ function createDirectoriesIfNotExist () {
73 tasks.push(ensureDir(dir)) 73 tasks.push(ensureDir(dir))
74 } 74 }
75 75
76 // Playlist directories
77 tasks.push(ensureDir(HLS_PLAYLIST_DIRECTORY))
78
76 return Promise.all(tasks) 79 return Promise.all(tasks)
77} 80}
78 81
diff --git a/server/initializers/migrations/0330-video-streaming-playlist.ts b/server/initializers/migrations/0330-video-streaming-playlist.ts
new file mode 100644
index 000000000..c85a762ab
--- /dev/null
+++ b/server/initializers/migrations/0330-video-streaming-playlist.ts
@@ -0,0 +1,51 @@
1import * as Sequelize from 'sequelize'
2
3async function up (utils: {
4 transaction: Sequelize.Transaction,
5 queryInterface: Sequelize.QueryInterface,
6 sequelize: Sequelize.Sequelize
7}): Promise<void> {
8
9 {
10 const query = `
11 CREATE TABLE IF NOT EXISTS "videoStreamingPlaylist"
12(
13 "id" SERIAL,
14 "type" INTEGER NOT NULL,
15 "playlistUrl" VARCHAR(2000) NOT NULL,
16 "p2pMediaLoaderInfohashes" VARCHAR(255)[] NOT NULL,
17 "segmentsSha256Url" VARCHAR(255) NOT NULL,
18 "videoId" INTEGER NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
19 "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL,
20 "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL,
21 PRIMARY KEY ("id")
22);`
23 await utils.sequelize.query(query)
24 }
25
26 {
27 const data = {
28 type: Sequelize.INTEGER,
29 allowNull: true,
30 defaultValue: null
31 }
32
33 await utils.queryInterface.changeColumn('videoRedundancy', 'videoFileId', data)
34 }
35
36 {
37 const query = 'ALTER TABLE "videoRedundancy" ADD COLUMN "videoStreamingPlaylistId" INTEGER NULL ' +
38 'REFERENCES "videoStreamingPlaylist" ("id") ON DELETE CASCADE ON UPDATE CASCADE'
39
40 await utils.sequelize.query(query)
41 }
42}
43
44function down (options) {
45 throw new Error('Not implemented.')
46}
47
48export {
49 up,
50 down
51}
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts
index f6f068b45..9a40414bb 100644
--- a/server/lib/activitypub/cache-file.ts
+++ b/server/lib/activitypub/cache-file.ts
@@ -1,11 +1,28 @@
1import { CacheFileObject } from '../../../shared/index' 1import { ActivityPlaylistUrlObject, ActivityVideoUrlObject, CacheFileObject } from '../../../shared/index'
2import { VideoModel } from '../../models/video/video' 2import { VideoModel } from '../../models/video/video'
3import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 3import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
4import { Transaction } from 'sequelize' 4import { Transaction } from 'sequelize'
5import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
5 6
6function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) { 7function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: { id?: number }) {
7 const url = cacheFileObject.url
8 8
9 if (cacheFileObject.url.mediaType === 'application/x-mpegURL') {
10 const url = cacheFileObject.url
11
12 const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS)
13 if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url)
14
15 return {
16 expiresOn: new Date(cacheFileObject.expires),
17 url: cacheFileObject.id,
18 fileUrl: url.href,
19 strategy: null,
20 videoStreamingPlaylistId: playlist.id,
21 actorId: byActor.id
22 }
23 }
24
25 const url = cacheFileObject.url
9 const videoFile = video.VideoFiles.find(f => { 26 const videoFile = video.VideoFiles.find(f => {
10 return f.resolution === url.height && f.fps === url.fps 27 return f.resolution === url.height && f.fps === url.fps
11 }) 28 })
@@ -15,7 +32,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
15 return { 32 return {
16 expiresOn: new Date(cacheFileObject.expires), 33 expiresOn: new Date(cacheFileObject.expires),
17 url: cacheFileObject.id, 34 url: cacheFileObject.id,
18 fileUrl: cacheFileObject.url.href, 35 fileUrl: url.href,
19 strategy: null, 36 strategy: null,
20 videoFileId: videoFile.id, 37 videoFileId: videoFile.id,
21 actorId: byActor.id 38 actorId: byActor.id
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts
index e3fca0a17..605aaba06 100644
--- a/server/lib/activitypub/send/send-create.ts
+++ b/server/lib/activitypub/send/send-create.ts
@@ -1,6 +1,6 @@
1import { Transaction } from 'sequelize' 1import { Transaction } from 'sequelize'
2import { ActivityAudience, ActivityCreate } from '../../../../shared/models/activitypub' 2import { ActivityAudience, ActivityCreate } from '../../../../shared/models/activitypub'
3import { VideoPrivacy } from '../../../../shared/models/videos' 3import { Video, VideoPrivacy } from '../../../../shared/models/videos'
4import { ActorModel } from '../../../models/activitypub/actor' 4import { ActorModel } from '../../../models/activitypub/actor'
5import { VideoModel } from '../../../models/video/video' 5import { VideoModel } from '../../../models/video/video'
6import { VideoAbuseModel } from '../../../models/video/video-abuse' 6import { VideoAbuseModel } from '../../../models/video/video-abuse'
@@ -39,17 +39,14 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel,
39 return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) 39 return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
40} 40}
41 41
42async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) { 42async function sendCreateCacheFile (byActor: ActorModel, video: VideoModel, fileRedundancy: VideoRedundancyModel) {
43 logger.info('Creating job to send file cache of %s.', fileRedundancy.url) 43 logger.info('Creating job to send file cache of %s.', fileRedundancy.url)
44 44
45 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id)
46 const redundancyObject = fileRedundancy.toActivityPubObject()
47
48 return sendVideoRelatedCreateActivity({ 45 return sendVideoRelatedCreateActivity({
49 byActor, 46 byActor,
50 video, 47 video,
51 url: fileRedundancy.url, 48 url: fileRedundancy.url,
52 object: redundancyObject 49 object: fileRedundancy.toActivityPubObject()
53 }) 50 })
54} 51}
55 52
diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts
index bf1b6e117..8976fcbc8 100644
--- a/server/lib/activitypub/send/send-undo.ts
+++ b/server/lib/activitypub/send/send-undo.ts
@@ -73,7 +73,8 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans
73async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { 73async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) {
74 logger.info('Creating job to undo cache file %s.', redundancyModel.url) 74 logger.info('Creating job to undo cache file %s.', redundancyModel.url)
75 75
76 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) 76 const videoId = redundancyModel.getVideo().id
77 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
77 const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) 78 const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject())
78 79
79 return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t }) 80 return sendUndoVideoRelatedActivity({ byActor, video, url: redundancyModel.url, activity: createActivity, transaction: t })
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts
index a68f03edf..839f66470 100644
--- a/server/lib/activitypub/send/send-update.ts
+++ b/server/lib/activitypub/send/send-update.ts
@@ -61,7 +61,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod
61async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { 61async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) {
62 logger.info('Creating job to update cache file %s.', redundancyModel.url) 62 logger.info('Creating job to update cache file %s.', redundancyModel.url)
63 63
64 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) 64 const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.getVideo().id)
65 65
66 const activityBuilder = (audience: ActivityAudience) => { 66 const activityBuilder = (audience: ActivityAudience) => {
67 const redundancyObject = redundancyModel.toActivityPubObject() 67 const redundancyObject = redundancyModel.toActivityPubObject()
diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts
index 38f15448c..4229fe094 100644
--- a/server/lib/activitypub/url.ts
+++ b/server/lib/activitypub/url.ts
@@ -5,6 +5,8 @@ import { VideoModel } from '../../models/video/video'
5import { VideoAbuseModel } from '../../models/video/video-abuse' 5import { VideoAbuseModel } from '../../models/video/video-abuse'
6import { VideoCommentModel } from '../../models/video/video-comment' 6import { VideoCommentModel } from '../../models/video/video-comment'
7import { VideoFileModel } from '../../models/video/video-file' 7import { VideoFileModel } from '../../models/video/video-file'
8import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model'
9import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
8 10
9function getVideoActivityPubUrl (video: VideoModel) { 11function getVideoActivityPubUrl (video: VideoModel) {
10 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid 12 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid
@@ -16,6 +18,10 @@ function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) {
16 return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` 18 return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}`
17} 19}
18 20
21function getVideoCacheStreamingPlaylistActivityPubUrl (video: VideoModel, playlist: VideoStreamingPlaylistModel) {
22 return `${CONFIG.WEBSERVER.URL}/redundancy/video-playlists/${playlist.getStringType()}/${video.uuid}`
23}
24
19function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { 25function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) {
20 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id 26 return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id
21} 27}
@@ -92,6 +98,7 @@ function getUndoActivityPubUrl (originalUrl: string) {
92 98
93export { 99export {
94 getVideoActivityPubUrl, 100 getVideoActivityPubUrl,
101 getVideoCacheStreamingPlaylistActivityPubUrl,
95 getVideoChannelActivityPubUrl, 102 getVideoChannelActivityPubUrl,
96 getAccountActivityPubUrl, 103 getAccountActivityPubUrl,
97 getVideoAbuseActivityPubUrl, 104 getVideoAbuseActivityPubUrl,
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index e1e523499..edd01234f 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -2,7 +2,14 @@ import * as Bluebird from 'bluebird'
2import * as sequelize from 'sequelize' 2import * as sequelize from 'sequelize'
3import * as magnetUtil from 'magnet-uri' 3import * as magnetUtil from 'magnet-uri'
4import * as request from 'request' 4import * as request from 'request'
5import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index' 5import {
6 ActivityIconObject,
7 ActivityPlaylistSegmentHashesObject,
8 ActivityPlaylistUrlObject,
9 ActivityUrlObject,
10 ActivityVideoUrlObject,
11 VideoState
12} from '../../../shared/index'
6import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' 13import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
7import { VideoPrivacy } from '../../../shared/models/videos' 14import { VideoPrivacy } from '../../../shared/models/videos'
8import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' 15import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
@@ -30,6 +37,9 @@ import { AccountModel } from '../../models/account/account'
30import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' 37import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
31import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' 38import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
32import { Notifier } from '../notifier' 39import { Notifier } from '../notifier'
40import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
41import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
42import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
33 43
34async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { 44async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
35 // If the video is not private and published, we federate it 45 // If the video is not private and published, we federate it
@@ -264,6 +274,25 @@ async function updateVideoFromAP (options: {
264 } 274 }
265 275
266 { 276 {
277 const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(options.video, options.videoObject)
278 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
279
280 // Remove video files that do not exist anymore
281 const destroyTasks = options.video.VideoStreamingPlaylists
282 .filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f)))
283 .map(f => f.destroy(sequelizeOptions))
284 await Promise.all(destroyTasks)
285
286 // Update or add other one
287 const upsertTasks = streamingPlaylistAttributes.map(a => {
288 return VideoStreamingPlaylistModel.upsert<VideoStreamingPlaylistModel>(a, { returning: true, transaction: t })
289 .then(([ streamingPlaylist ]) => streamingPlaylist)
290 })
291
292 options.video.VideoStreamingPlaylists = await Promise.all(upsertTasks)
293 }
294
295 {
267 // Update Tags 296 // Update Tags
268 const tags = options.videoObject.tag.map(tag => tag.name) 297 const tags = options.videoObject.tag.map(tag => tag.name)
269 const tagInstances = await TagModel.findOrCreateTags(tags, t) 298 const tagInstances = await TagModel.findOrCreateTags(tags, t)
@@ -367,13 +396,25 @@ export {
367 396
368// --------------------------------------------------------------------------- 397// ---------------------------------------------------------------------------
369 398
370function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { 399function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
371 const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) 400 const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)
372 401
373 const urlMediaType = url.mediaType || url.mimeType 402 const urlMediaType = url.mediaType || url.mimeType
374 return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') 403 return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/')
375} 404}
376 405
406function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject {
407 const urlMediaType = url.mediaType || url.mimeType
408
409 return urlMediaType === 'application/x-mpegURL'
410}
411
412function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
413 const urlMediaType = tag.mediaType || tag.mimeType
414
415 return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json'
416}
417
377async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { 418async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) {
378 logger.debug('Adding remote video %s.', videoObject.id) 419 logger.debug('Adding remote video %s.', videoObject.id)
379 420
@@ -394,8 +435,14 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor
394 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) 435 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
395 await Promise.all(videoFilePromises) 436 await Promise.all(videoFilePromises)
396 437
438 const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject)
439 const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t }))
440 await Promise.all(playlistPromises)
441
397 // Process tags 442 // Process tags
398 const tags = videoObject.tag.map(t => t.name) 443 const tags = videoObject.tag
444 .filter(t => t.type === 'Hashtag')
445 .map(t => t.name)
399 const tagInstances = await TagModel.findOrCreateTags(tags, t) 446 const tagInstances = await TagModel.findOrCreateTags(tags, t)
400 await videoCreated.$set('Tags', tagInstances, sequelizeOptions) 447 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
401 448
@@ -473,13 +520,13 @@ async function videoActivityObjectToDBAttributes (
473} 520}
474 521
475function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) { 522function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) {
476 const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] 523 const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
477 524
478 if (fileUrls.length === 0) { 525 if (fileUrls.length === 0) {
479 throw new Error('Cannot find video files for ' + video.url) 526 throw new Error('Cannot find video files for ' + video.url)
480 } 527 }
481 528
482 const attributes: VideoFileModel[] = [] 529 const attributes: FilteredModelAttributes<VideoFileModel>[] = []
483 for (const fileUrl of fileUrls) { 530 for (const fileUrl of fileUrls) {
484 // Fetch associated magnet uri 531 // Fetch associated magnet uri
485 const magnet = videoObject.url.find(u => { 532 const magnet = videoObject.url.find(u => {
@@ -502,7 +549,45 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid
502 size: fileUrl.size, 549 size: fileUrl.size,
503 videoId: video.id, 550 videoId: video.id,
504 fps: fileUrl.fps || -1 551 fps: fileUrl.fps || -1
505 } as VideoFileModel 552 }
553
554 attributes.push(attribute)
555 }
556
557 return attributes
558}
559
560function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) {
561 const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
562 if (playlistUrls.length === 0) return []
563
564 const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = []
565 for (const playlistUrlObject of playlistUrls) {
566 const p2pMediaLoaderInfohashes = playlistUrlObject.tag
567 .filter(t => t.type === 'Infohash')
568 .map(t => t.name)
569 if (p2pMediaLoaderInfohashes.length === 0) {
570 logger.warn('No infohashes found in AP playlist object.', { playlistUrl: playlistUrlObject })
571 continue
572 }
573
574 const segmentsSha256UrlObject = playlistUrlObject.tag
575 .find(t => {
576 return isAPPlaylistSegmentHashesUrlObject(t)
577 }) as ActivityPlaylistSegmentHashesObject
578 if (!segmentsSha256UrlObject) {
579 logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
580 continue
581 }
582
583 const attribute = {
584 type: VideoStreamingPlaylistType.HLS,
585 playlistUrl: playlistUrlObject.href,
586 segmentsSha256Url: segmentsSha256UrlObject.href,
587 p2pMediaLoaderInfohashes,
588 videoId: video.id
589 }
590
506 attributes.push(attribute) 591 attributes.push(attribute)
507 } 592 }
508 593
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
new file mode 100644
index 000000000..10db6c3c3
--- /dev/null
+++ b/server/lib/hls.ts
@@ -0,0 +1,110 @@
1import { VideoModel } from '../models/video/video'
2import { basename, dirname, join } from 'path'
3import { HLS_PLAYLIST_DIRECTORY, CONFIG } from '../initializers'
4import { outputJSON, pathExists, readdir, readFile, remove, writeFile, move } from 'fs-extra'
5import { getVideoFileSize } from '../helpers/ffmpeg-utils'
6import { sha256 } from '../helpers/core-utils'
7import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
8import HLSDownloader from 'hlsdownloader'
9import { logger } from '../helpers/logger'
10import { parse } from 'url'
11
12async function updateMasterHLSPlaylist (video: VideoModel) {
13 const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
14 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
15 const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
16
17 for (const file of video.VideoFiles) {
18 // If we did not generated a playlist for this resolution, skip
19 const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
20 if (await pathExists(filePlaylistPath) === false) continue
21
22 const videoFilePath = video.getVideoFilePath(file)
23
24 const size = await getVideoFileSize(videoFilePath)
25
26 const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file)
27 const resolution = `RESOLUTION=${size.width}x${size.height}`
28
29 let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}`
30 if (file.fps) line += ',FRAME-RATE=' + file.fps
31
32 masterPlaylists.push(line)
33 masterPlaylists.push(VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
34 }
35
36 await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')
37}
38
39async function updateSha256Segments (video: VideoModel) {
40 const directory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
41 const files = await readdir(directory)
42 const json: { [filename: string]: string} = {}
43
44 for (const file of files) {
45 if (file.endsWith('.ts') === false) continue
46
47 const buffer = await readFile(join(directory, file))
48 const filename = basename(file)
49
50 json[filename] = sha256(buffer)
51 }
52
53 const outputPath = join(directory, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
54 await outputJSON(outputPath, json)
55}
56
57function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number) {
58 let timer
59
60 logger.info('Importing HLS playlist %s', playlistUrl)
61
62 const params = {
63 playlistURL: playlistUrl,
64 destination: CONFIG.STORAGE.TMP_DIR
65 }
66 const downloader = new HLSDownloader(params)
67
68 const hlsDestinationDir = join(CONFIG.STORAGE.TMP_DIR, dirname(parse(playlistUrl).pathname))
69
70 return new Promise<string>(async (res, rej) => {
71 downloader.startDownload(err => {
72 clearTimeout(timer)
73
74 if (err) {
75 deleteTmpDirectory(hlsDestinationDir)
76
77 return rej(err)
78 }
79
80 move(hlsDestinationDir, destinationDir, { overwrite: true })
81 .then(() => res())
82 .catch(err => {
83 deleteTmpDirectory(hlsDestinationDir)
84
85 return rej(err)
86 })
87 })
88
89 timer = setTimeout(() => {
90 deleteTmpDirectory(hlsDestinationDir)
91
92 return rej(new Error('HLS download timeout.'))
93 }, timeout)
94
95 function deleteTmpDirectory (directory: string) {
96 remove(directory)
97 .catch(err => logger.error('Cannot delete path on HLS download error.', { err }))
98 }
99 })
100}
101
102// ---------------------------------------------------------------------------
103
104export {
105 updateMasterHLSPlaylist,
106 updateSha256Segments,
107 downloadPlaylistSegments
108}
109
110// ---------------------------------------------------------------------------
diff --git a/server/lib/job-queue/handlers/video-file.ts b/server/lib/job-queue/handlers/video-file.ts
index 217d666b6..7119ce0ca 100644
--- a/server/lib/job-queue/handlers/video-file.ts
+++ b/server/lib/job-queue/handlers/video-file.ts
@@ -5,17 +5,18 @@ import { VideoModel } from '../../../models/video/video'
5import { JobQueue } from '../job-queue' 5import { JobQueue } from '../job-queue'
6import { federateVideoIfNeeded } from '../../activitypub' 6import { federateVideoIfNeeded } from '../../activitypub'
7import { retryTransactionWrapper } from '../../../helpers/database-utils' 7import { retryTransactionWrapper } from '../../../helpers/database-utils'
8import { sequelizeTypescript } from '../../../initializers' 8import { sequelizeTypescript, CONFIG } from '../../../initializers'
9import * as Bluebird from 'bluebird' 9import * as Bluebird from 'bluebird'
10import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' 10import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils'
11import { importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding' 11import { generateHlsPlaylist, importVideoFile, optimizeVideofile, transcodeOriginalVideofile } from '../../video-transcoding'
12import { Notifier } from '../../notifier' 12import { Notifier } from '../../notifier'
13 13
14export type VideoFilePayload = { 14export type VideoFilePayload = {
15 videoUUID: string 15 videoUUID: string
16 isNewVideo?: boolean
17 resolution?: VideoResolution 16 resolution?: VideoResolution
17 isNewVideo?: boolean
18 isPortraitMode?: boolean 18 isPortraitMode?: boolean
19 generateHlsPlaylist?: boolean
19} 20}
20 21
21export type VideoFileImportPayload = { 22export type VideoFileImportPayload = {
@@ -51,21 +52,38 @@ async function processVideoFile (job: Bull.Job) {
51 return undefined 52 return undefined
52 } 53 }
53 54
54 // Transcoding in other resolution 55 if (payload.generateHlsPlaylist) {
55 if (payload.resolution) { 56 await generateHlsPlaylist(video, payload.resolution, payload.isPortraitMode || false)
57
58 await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video)
59 } else if (payload.resolution) { // Transcoding in other resolution
56 await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) 60 await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false)
57 61
58 await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video) 62 await retryTransactionWrapper(onVideoFileTranscoderOrImportSuccess, video, payload)
59 } else { 63 } else {
60 await optimizeVideofile(video) 64 await optimizeVideofile(video)
61 65
62 await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload.isNewVideo) 66 await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload)
63 } 67 }
64 68
65 return video 69 return video
66} 70}
67 71
68async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) { 72async function onHlsPlaylistGenerationSuccess (video: VideoModel) {
73 if (video === undefined) return undefined
74
75 await sequelizeTypescript.transaction(async t => {
76 // Maybe the video changed in database, refresh it
77 let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
78 // Video does not exist anymore
79 if (!videoDatabase) return undefined
80
81 // If the video was not published, we consider it is a new one for other instances
82 await federateVideoIfNeeded(videoDatabase, false, t)
83 })
84}
85
86async function onVideoFileTranscoderOrImportSuccess (video: VideoModel, payload?: VideoFilePayload) {
69 if (video === undefined) return undefined 87 if (video === undefined) return undefined
70 88
71 const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { 89 const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => {
@@ -96,9 +114,11 @@ async function onVideoFileTranscoderOrImportSuccess (video: VideoModel) {
96 Notifier.Instance.notifyOnNewVideo(videoDatabase) 114 Notifier.Instance.notifyOnNewVideo(videoDatabase)
97 Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) 115 Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase)
98 } 116 }
117
118 await createHlsJobIfEnabled(payload)
99} 119}
100 120
101async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: boolean) { 121async function onVideoFileOptimizerSuccess (videoArg: VideoModel, payload: VideoFilePayload) {
102 if (videoArg === undefined) return undefined 122 if (videoArg === undefined) return undefined
103 123
104 // Outside the transaction (IO on disk) 124 // Outside the transaction (IO on disk)
@@ -145,7 +165,7 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo
145 logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy }) 165 logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy })
146 } 166 }
147 167
148 await federateVideoIfNeeded(videoDatabase, isNewVideo, t) 168 await federateVideoIfNeeded(videoDatabase, payload.isNewVideo, t)
149 169
150 return { videoDatabase, videoPublished } 170 return { videoDatabase, videoPublished }
151 }) 171 })
@@ -155,6 +175,8 @@ async function onVideoFileOptimizerSuccess (videoArg: VideoModel, isNewVideo: bo
155 if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase) 175 if (isNewVideo) Notifier.Instance.notifyOnNewVideo(videoDatabase)
156 if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase) 176 if (videoPublished) Notifier.Instance.notifyOnPendingVideoPublished(videoDatabase)
157 } 177 }
178
179 await createHlsJobIfEnabled(Object.assign({}, payload, { resolution: videoDatabase.getOriginalFile().resolution }))
158} 180}
159 181
160// --------------------------------------------------------------------------- 182// ---------------------------------------------------------------------------
@@ -163,3 +185,20 @@ export {
163 processVideoFile, 185 processVideoFile,
164 processVideoFileImport 186 processVideoFileImport
165} 187}
188
189// ---------------------------------------------------------------------------
190
191function createHlsJobIfEnabled (payload?: VideoFilePayload) {
192 // Generate HLS playlist?
193 if (payload && CONFIG.TRANSCODING.HLS.ENABLED) {
194 const hlsTranscodingPayload = {
195 videoUUID: payload.videoUUID,
196 resolution: payload.resolution,
197 isPortraitMode: payload.isPortraitMode,
198
199 generateHlsPlaylist: true
200 }
201
202 return JobQueue.Instance.createJob({ type: 'video-file', payload: hlsTranscodingPayload })
203 }
204}
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index f643ee226..1a48f2bd0 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,5 +1,5 @@
1import { AbstractScheduler } from './abstract-scheduler' 1import { AbstractScheduler } from './abstract-scheduler'
2import { CONFIG, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers' 2import { CONFIG, HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { VideosRedundancy } from '../../../shared/models/redundancy' 4import { VideosRedundancy } from '../../../shared/models/redundancy'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
@@ -9,9 +9,19 @@ import { join } from 'path'
9import { move } from 'fs-extra' 9import { move } from 'fs-extra'
10import { getServerActor } from '../../helpers/utils' 10import { getServerActor } from '../../helpers/utils'
11import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' 11import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
12import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' 12import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url'
13import { removeVideoRedundancy } from '../redundancy' 13import { removeVideoRedundancy } from '../redundancy'
14import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' 14import { getOrCreateVideoAndAccountAndChannel } from '../activitypub'
15import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
16import { VideoModel } from '../../models/video/video'
17import { downloadPlaylistSegments } from '../hls'
18
19type CandidateToDuplicate = {
20 redundancy: VideosRedundancy,
21 video: VideoModel,
22 files: VideoFileModel[],
23 streamingPlaylists: VideoStreamingPlaylistModel[]
24}
15 25
16export class VideosRedundancyScheduler extends AbstractScheduler { 26export class VideosRedundancyScheduler extends AbstractScheduler {
17 27
@@ -24,28 +34,32 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
24 } 34 }
25 35
26 protected async internalExecute () { 36 protected async internalExecute () {
27 for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { 37 for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
28 logger.info('Running redundancy scheduler for strategy %s.', obj.strategy) 38 logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy)
29 39
30 try { 40 try {
31 const videoToDuplicate = await this.findVideoToDuplicate(obj) 41 const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig)
32 if (!videoToDuplicate) continue 42 if (!videoToDuplicate) continue
33 43
34 const videoFiles = videoToDuplicate.VideoFiles 44 const candidateToDuplicate = {
35 videoFiles.forEach(f => f.Video = videoToDuplicate) 45 video: videoToDuplicate,
46 redundancy: redundancyConfig,
47 files: videoToDuplicate.VideoFiles,
48 streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists
49 }
36 50
37 await this.purgeCacheIfNeeded(obj, videoFiles) 51 await this.purgeCacheIfNeeded(candidateToDuplicate)
38 52
39 if (await this.isTooHeavy(obj, videoFiles)) { 53 if (await this.isTooHeavy(candidateToDuplicate)) {
40 logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) 54 logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url)
41 continue 55 continue
42 } 56 }
43 57
44 logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy) 58 logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, redundancyConfig.strategy)
45 59
46 await this.createVideoRedundancy(obj, videoFiles) 60 await this.createVideoRedundancies(candidateToDuplicate)
47 } catch (err) { 61 } catch (err) {
48 logger.error('Cannot run videos redundancy %s.', obj.strategy, { err }) 62 logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err })
49 } 63 }
50 } 64 }
51 65
@@ -63,25 +77,35 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
63 77
64 for (const redundancyModel of expired) { 78 for (const redundancyModel of expired) {
65 try { 79 try {
66 await this.extendsOrDeleteRedundancy(redundancyModel) 80 const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
81 const candidate = {
82 redundancy: redundancyConfig,
83 video: null,
84 files: [],
85 streamingPlaylists: []
86 }
87
88 // If the administrator disabled the redundancy or decreased the cache size, remove this redundancy instead of extending it
89 if (!redundancyConfig || await this.isTooHeavy(candidate)) {
90 logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy)
91 await removeVideoRedundancy(redundancyModel)
92 } else {
93 await this.extendsRedundancy(redundancyModel)
94 }
67 } catch (err) { 95 } catch (err) {
68 logger.error('Cannot extend expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel)) 96 logger.error(
97 'Cannot extend or remove expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel),
98 { err }
99 )
69 } 100 }
70 } 101 }
71 } 102 }
72 103
73 private async extendsOrDeleteRedundancy (redundancyModel: VideoRedundancyModel) { 104 private async extendsRedundancy (redundancyModel: VideoRedundancyModel) {
74 // Refresh the video, maybe it was deleted
75 const video = await this.loadAndRefreshVideo(redundancyModel.VideoFile.Video.url)
76
77 if (!video) {
78 logger.info('Destroying existing redundancy %s, because the associated video does not exist anymore.', redundancyModel.url)
79
80 await redundancyModel.destroy()
81 return
82 }
83
84 const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) 105 const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
106 // Redundancy strategy disabled, remove our redundancy instead of extending expiration
107 if (!redundancy) await removeVideoRedundancy(redundancyModel)
108
85 await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) 109 await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime)
86 } 110 }
87 111
@@ -112,49 +136,93 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
112 } 136 }
113 } 137 }
114 138
115 private async createVideoRedundancy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { 139 private async createVideoRedundancies (data: CandidateToDuplicate) {
116 const serverActor = await getServerActor() 140 const video = await this.loadAndRefreshVideo(data.video.url)
141
142 if (!video) {
143 logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url)
117 144
118 for (const file of filesToDuplicate) { 145 return
119 const video = await this.loadAndRefreshVideo(file.Video.url) 146 }
120 147
148 for (const file of data.files) {
121 const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) 149 const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
122 if (existingRedundancy) { 150 if (existingRedundancy) {
123 await this.extendsOrDeleteRedundancy(existingRedundancy) 151 await this.extendsRedundancy(existingRedundancy)
124 152
125 continue 153 continue
126 } 154 }
127 155
128 if (!video) { 156 await this.createVideoFileRedundancy(data.redundancy, video, file)
129 logger.info('Video %s we want to duplicate does not existing anymore, skipping.', file.Video.url) 157 }
158
159 for (const streamingPlaylist of data.streamingPlaylists) {
160 const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id)
161 if (existingRedundancy) {
162 await this.extendsRedundancy(existingRedundancy)
130 163
131 continue 164 continue
132 } 165 }
133 166
134 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) 167 await this.createStreamingPlaylistRedundancy(data.redundancy, video, streamingPlaylist)
168 }
169 }
135 170
136 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() 171 private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: VideoModel, file: VideoFileModel) {
137 const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) 172 file.Video = video
138 173
139 const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) 174 const serverActor = await getServerActor()
140 175
141 const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) 176 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy)
142 await move(tmpPath, destPath)
143 177
144 const createdModel = await VideoRedundancyModel.create({ 178 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
145 expiresOn: this.buildNewExpiration(redundancy.minLifetime), 179 const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs)
146 url: getVideoCacheFileActivityPubUrl(file),
147 fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL),
148 strategy: redundancy.strategy,
149 videoFileId: file.id,
150 actorId: serverActor.id
151 })
152 createdModel.VideoFile = file
153 180
154 await sendCreateCacheFile(serverActor, createdModel) 181 const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT)
155 182
156 logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url) 183 const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file))
157 } 184 await move(tmpPath, destPath)
185
186 const createdModel = await VideoRedundancyModel.create({
187 expiresOn: this.buildNewExpiration(redundancy.minLifetime),
188 url: getVideoCacheFileActivityPubUrl(file),
189 fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL),
190 strategy: redundancy.strategy,
191 videoFileId: file.id,
192 actorId: serverActor.id
193 })
194
195 createdModel.VideoFile = file
196
197 await sendCreateCacheFile(serverActor, video, createdModel)
198
199 logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url)
200 }
201
202 private async createStreamingPlaylistRedundancy (redundancy: VideosRedundancy, video: VideoModel, playlist: VideoStreamingPlaylistModel) {
203 playlist.Video = video
204
205 const serverActor = await getServerActor()
206
207 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy)
208
209 const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
210 await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
211
212 const createdModel = await VideoRedundancyModel.create({
213 expiresOn: this.buildNewExpiration(redundancy.minLifetime),
214 url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist),
215 fileUrl: playlist.getVideoRedundancyUrl(CONFIG.WEBSERVER.URL),
216 strategy: redundancy.strategy,
217 videoStreamingPlaylistId: playlist.id,
218 actorId: serverActor.id
219 })
220
221 createdModel.VideoStreamingPlaylist = playlist
222
223 await sendCreateCacheFile(serverActor, video, createdModel)
224
225 logger.info('Duplicated playlist %s -> %s.', playlist.playlistUrl, createdModel.url)
158 } 226 }
159 227
160 private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) { 228 private async extendsExpirationOf (redundancy: VideoRedundancyModel, expiresAfterMs: number) {
@@ -168,8 +236,9 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
168 await sendUpdateCacheFile(serverActor, redundancy) 236 await sendUpdateCacheFile(serverActor, redundancy)
169 } 237 }
170 238
171 private async purgeCacheIfNeeded (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { 239 private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) {
172 while (this.isTooHeavy(redundancy, filesToDuplicate)) { 240 while (this.isTooHeavy(candidateToDuplicate)) {
241 const redundancy = candidateToDuplicate.redundancy
173 const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime) 242 const toDelete = await VideoRedundancyModel.loadOldestLocalThatAlreadyExpired(redundancy.strategy, redundancy.minLifetime)
174 if (!toDelete) return 243 if (!toDelete) return
175 244
@@ -177,11 +246,11 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
177 } 246 }
178 } 247 }
179 248
180 private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) { 249 private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) {
181 const maxSize = redundancy.size 250 const maxSize = candidateToDuplicate.redundancy.size
182 251
183 const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy) 252 const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(candidateToDuplicate.redundancy.strategy)
184 const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(filesToDuplicate) 253 const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists)
185 254
186 return totalWillDuplicate > maxSize 255 return totalWillDuplicate > maxSize
187 } 256 }
@@ -191,13 +260,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
191 } 260 }
192 261
193 private buildEntryLogId (object: VideoRedundancyModel) { 262 private buildEntryLogId (object: VideoRedundancyModel) {
194 return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` 263 if (object.VideoFile) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}`
264
265 return `${object.VideoStreamingPlaylist.playlistUrl}`
195 } 266 }
196 267
197 private getTotalFileSizes (files: VideoFileModel[]) { 268 private getTotalFileSizes (files: VideoFileModel[], playlists: VideoStreamingPlaylistModel[]) {
198 const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size 269 const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size
199 270
200 return files.reduce(fileReducer, 0) 271 return files.reduce(fileReducer, 0) * playlists.length
201 } 272 }
202 273
203 private async loadAndRefreshVideo (videoUrl: string) { 274 private async loadAndRefreshVideo (videoUrl: string) {
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts
index 4460f46e4..608badfef 100644
--- a/server/lib/video-transcoding.ts
+++ b/server/lib/video-transcoding.ts
@@ -1,11 +1,14 @@
1import { CONFIG } from '../initializers' 1import { CONFIG, HLS_PLAYLIST_DIRECTORY } from '../initializers'
2import { extname, join } from 'path' 2import { extname, join } from 'path'
3import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils' 3import { getVideoFileFPS, getVideoFileResolution, transcode } from '../helpers/ffmpeg-utils'
4import { copy, remove, move, stat } from 'fs-extra' 4import { copy, ensureDir, move, remove, stat } from 'fs-extra'
5import { logger } from '../helpers/logger' 5import { logger } from '../helpers/logger'
6import { VideoResolution } from '../../shared/models/videos' 6import { VideoResolution } from '../../shared/models/videos'
7import { VideoFileModel } from '../models/video/video-file' 7import { VideoFileModel } from '../models/video/video-file'
8import { VideoModel } from '../models/video/video' 8import { VideoModel } from '../models/video/video'
9import { updateMasterHLSPlaylist, updateSha256Segments } from './hls'
10import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
11import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type'
9 12
10async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) { 13async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFileModel) {
11 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR 14 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
@@ -17,7 +20,8 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi
17 20
18 const transcodeOptions = { 21 const transcodeOptions = {
19 inputPath: videoInputPath, 22 inputPath: videoInputPath,
20 outputPath: videoTranscodedPath 23 outputPath: videoTranscodedPath,
24 resolution: inputVideoFile.resolution
21 } 25 }
22 26
23 // Could be very long! 27 // Could be very long!
@@ -47,7 +51,7 @@ async function optimizeVideofile (video: VideoModel, inputVideoFileArg?: VideoFi
47 } 51 }
48} 52}
49 53
50async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) { 54async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoResolution, isPortrait: boolean) {
51 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR 55 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
52 const extname = '.mp4' 56 const extname = '.mp4'
53 57
@@ -60,13 +64,13 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR
60 size: 0, 64 size: 0,
61 videoId: video.id 65 videoId: video.id
62 }) 66 })
63 const videoOutputPath = join(videosDirectory, video.getVideoFilename(newVideoFile)) 67 const videoOutputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(newVideoFile))
64 68
65 const transcodeOptions = { 69 const transcodeOptions = {
66 inputPath: videoInputPath, 70 inputPath: videoInputPath,
67 outputPath: videoOutputPath, 71 outputPath: videoOutputPath,
68 resolution, 72 resolution,
69 isPortraitMode 73 isPortraitMode: isPortrait
70 } 74 }
71 75
72 await transcode(transcodeOptions) 76 await transcode(transcodeOptions)
@@ -84,6 +88,38 @@ async function transcodeOriginalVideofile (video: VideoModel, resolution: VideoR
84 video.VideoFiles.push(newVideoFile) 88 video.VideoFiles.push(newVideoFile)
85} 89}
86 90
91async function generateHlsPlaylist (video: VideoModel, resolution: VideoResolution, isPortraitMode: boolean) {
92 const baseHlsDirectory = join(HLS_PLAYLIST_DIRECTORY, video.uuid)
93 await ensureDir(join(HLS_PLAYLIST_DIRECTORY, video.uuid))
94
95 const videoInputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(video.getOriginalFile()))
96 const outputPath = join(baseHlsDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution))
97
98 const transcodeOptions = {
99 inputPath: videoInputPath,
100 outputPath,
101 resolution,
102 isPortraitMode,
103 generateHlsPlaylist: true
104 }
105
106 await transcode(transcodeOptions)
107
108 await updateMasterHLSPlaylist(video)
109 await updateSha256Segments(video)
110
111 const playlistUrl = CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
112
113 await VideoStreamingPlaylistModel.upsert({
114 videoId: video.id,
115 playlistUrl,
116 segmentsSha256Url: CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid),
117 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrl, video.VideoFiles),
118
119 type: VideoStreamingPlaylistType.HLS
120 })
121}
122
87async function importVideoFile (video: VideoModel, inputFilePath: string) { 123async function importVideoFile (video: VideoModel, inputFilePath: string) {
88 const { videoFileResolution } = await getVideoFileResolution(inputFilePath) 124 const { videoFileResolution } = await getVideoFileResolution(inputFilePath)
89 const { size } = await stat(inputFilePath) 125 const { size } = await stat(inputFilePath)
@@ -125,6 +161,7 @@ async function importVideoFile (video: VideoModel, inputFilePath: string) {
125} 161}
126 162
127export { 163export {
164 generateHlsPlaylist,
128 optimizeVideofile, 165 optimizeVideofile,
129 transcodeOriginalVideofile, 166 transcodeOriginalVideofile,
130 importVideoFile 167 importVideoFile
diff --git a/server/middlewares/validators/redundancy.ts b/server/middlewares/validators/redundancy.ts
index c72ab78b2..329322509 100644
--- a/server/middlewares/validators/redundancy.ts
+++ b/server/middlewares/validators/redundancy.ts
@@ -13,7 +13,7 @@ import { ActorFollowModel } from '../../models/activitypub/actor-follow'
13import { SERVER_ACTOR_NAME } from '../../initializers' 13import { SERVER_ACTOR_NAME } from '../../initializers'
14import { ServerModel } from '../../models/server/server' 14import { ServerModel } from '../../models/server/server'
15 15
16const videoRedundancyGetValidator = [ 16const videoFileRedundancyGetValidator = [
17 param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), 17 param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'),
18 param('resolution') 18 param('resolution')
19 .customSanitizer(toIntOrNull) 19 .customSanitizer(toIntOrNull)
@@ -24,7 +24,7 @@ const videoRedundancyGetValidator = [
24 .custom(exists).withMessage('Should have a valid fps'), 24 .custom(exists).withMessage('Should have a valid fps'),
25 25
26 async (req: express.Request, res: express.Response, next: express.NextFunction) => { 26 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
27 logger.debug('Checking videoRedundancyGetValidator parameters', { parameters: req.params }) 27 logger.debug('Checking videoFileRedundancyGetValidator parameters', { parameters: req.params })
28 28
29 if (areValidationErrors(req, res)) return 29 if (areValidationErrors(req, res)) return
30 if (!await isVideoExist(req.params.videoId, res)) return 30 if (!await isVideoExist(req.params.videoId, res)) return
@@ -38,7 +38,31 @@ const videoRedundancyGetValidator = [
38 res.locals.videoFile = videoFile 38 res.locals.videoFile = videoFile
39 39
40 const videoRedundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id) 40 const videoRedundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id)
41 if (!videoRedundancy)return res.status(404).json({ error: 'Video redundancy not found.' }) 41 if (!videoRedundancy) return res.status(404).json({ error: 'Video redundancy not found.' })
42 res.locals.videoRedundancy = videoRedundancy
43
44 return next()
45 }
46]
47
48const videoPlaylistRedundancyGetValidator = [
49 param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'),
50 param('streamingPlaylistType').custom(exists).withMessage('Should have a valid streaming playlist type'),
51
52 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
53 logger.debug('Checking videoPlaylistRedundancyGetValidator parameters', { parameters: req.params })
54
55 if (areValidationErrors(req, res)) return
56 if (!await isVideoExist(req.params.videoId, res)) return
57
58 const video: VideoModel = res.locals.video
59 const videoStreamingPlaylist = video.VideoStreamingPlaylists.find(p => p === req.params.streamingPlaylistType)
60
61 if (!videoStreamingPlaylist) return res.status(404).json({ error: 'Video playlist not found.' })
62 res.locals.videoStreamingPlaylist = videoStreamingPlaylist
63
64 const videoRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(videoStreamingPlaylist.id)
65 if (!videoRedundancy) return res.status(404).json({ error: 'Video redundancy not found.' })
42 res.locals.videoRedundancy = videoRedundancy 66 res.locals.videoRedundancy = videoRedundancy
43 67
44 return next() 68 return next()
@@ -75,6 +99,7 @@ const updateServerRedundancyValidator = [
75// --------------------------------------------------------------------------- 99// ---------------------------------------------------------------------------
76 100
77export { 101export {
78 videoRedundancyGetValidator, 102 videoFileRedundancyGetValidator,
103 videoPlaylistRedundancyGetValidator,
79 updateServerRedundancyValidator 104 updateServerRedundancyValidator
80} 105}
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index 8f2ef2d9a..b722bed14 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -28,6 +28,7 @@ import { sample } from 'lodash'
28import { isTestInstance } from '../../helpers/core-utils' 28import { isTestInstance } from '../../helpers/core-utils'
29import * as Bluebird from 'bluebird' 29import * as Bluebird from 'bluebird'
30import * as Sequelize from 'sequelize' 30import * as Sequelize from 'sequelize'
31import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
31 32
32export enum ScopeNames { 33export enum ScopeNames {
33 WITH_VIDEO = 'WITH_VIDEO' 34 WITH_VIDEO = 'WITH_VIDEO'
@@ -38,7 +39,17 @@ export enum ScopeNames {
38 include: [ 39 include: [
39 { 40 {
40 model: () => VideoFileModel, 41 model: () => VideoFileModel,
41 required: true, 42 required: false,
43 include: [
44 {
45 model: () => VideoModel,
46 required: true
47 }
48 ]
49 },
50 {
51 model: () => VideoStreamingPlaylistModel,
52 required: false,
42 include: [ 53 include: [
43 { 54 {
44 model: () => VideoModel, 55 model: () => VideoModel,
@@ -97,12 +108,24 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
97 108
98 @BelongsTo(() => VideoFileModel, { 109 @BelongsTo(() => VideoFileModel, {
99 foreignKey: { 110 foreignKey: {
100 allowNull: false 111 allowNull: true
101 }, 112 },
102 onDelete: 'cascade' 113 onDelete: 'cascade'
103 }) 114 })
104 VideoFile: VideoFileModel 115 VideoFile: VideoFileModel
105 116
117 @ForeignKey(() => VideoStreamingPlaylistModel)
118 @Column
119 videoStreamingPlaylistId: number
120
121 @BelongsTo(() => VideoStreamingPlaylistModel, {
122 foreignKey: {
123 allowNull: true
124 },
125 onDelete: 'cascade'
126 })
127 VideoStreamingPlaylist: VideoStreamingPlaylistModel
128
106 @ForeignKey(() => ActorModel) 129 @ForeignKey(() => ActorModel)
107 @Column 130 @Column
108 actorId: number 131 actorId: number
@@ -119,13 +142,25 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
119 static async removeFile (instance: VideoRedundancyModel) { 142 static async removeFile (instance: VideoRedundancyModel) {
120 if (!instance.isOwned()) return 143 if (!instance.isOwned()) return
121 144
122 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId) 145 if (instance.videoFileId) {
146 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
123 147
124 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}` 148 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
125 logger.info('Removing duplicated video file %s.', logIdentifier) 149 logger.info('Removing duplicated video file %s.', logIdentifier)
126 150
127 videoFile.Video.removeFile(videoFile, true) 151 videoFile.Video.removeFile(videoFile, true)
128 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err })) 152 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
153 }
154
155 if (instance.videoStreamingPlaylistId) {
156 const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
157
158 const videoUUID = videoStreamingPlaylist.Video.uuid
159 logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
160
161 videoStreamingPlaylist.Video.removeStreamingPlaylist(true)
162 .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
163 }
129 164
130 return undefined 165 return undefined
131 } 166 }
@@ -143,6 +178,19 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
143 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) 178 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
144 } 179 }
145 180
181 static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number) {
182 const actor = await getServerActor()
183
184 const query = {
185 where: {
186 actorId: actor.id,
187 videoStreamingPlaylistId
188 }
189 }
190
191 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
192 }
193
146 static loadByUrl (url: string, transaction?: Sequelize.Transaction) { 194 static loadByUrl (url: string, transaction?: Sequelize.Transaction) {
147 const query = { 195 const query = {
148 where: { 196 where: {
@@ -191,7 +239,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
191 const ids = rows.map(r => r.id) 239 const ids = rows.map(r => r.id)
192 const id = sample(ids) 240 const id = sample(ids)
193 241
194 return VideoModel.loadWithFile(id, undefined, !isTestInstance()) 242 return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
195 } 243 }
196 244
197 static async findMostViewToDuplicate (randomizedFactor: number) { 245 static async findMostViewToDuplicate (randomizedFactor: number) {
@@ -333,40 +381,44 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
333 381
334 static async listLocalOfServer (serverId: number) { 382 static async listLocalOfServer (serverId: number) {
335 const actor = await getServerActor() 383 const actor = await getServerActor()
336 384 const buildVideoInclude = () => ({
337 const query = { 385 model: VideoModel,
338 where: { 386 required: true,
339 actorId: actor.id
340 },
341 include: [ 387 include: [
342 { 388 {
343 model: VideoFileModel, 389 attributes: [],
390 model: VideoChannelModel.unscoped(),
344 required: true, 391 required: true,
345 include: [ 392 include: [
346 { 393 {
347 model: VideoModel, 394 attributes: [],
395 model: ActorModel.unscoped(),
348 required: true, 396 required: true,
349 include: [ 397 where: {
350 { 398 serverId
351 attributes: [], 399 }
352 model: VideoChannelModel.unscoped(),
353 required: true,
354 include: [
355 {
356 attributes: [],
357 model: ActorModel.unscoped(),
358 required: true,
359 where: {
360 serverId
361 }
362 }
363 ]
364 }
365 ]
366 } 400 }
367 ] 401 ]
368 } 402 }
369 ] 403 ]
404 })
405
406 const query = {
407 where: {
408 actorId: actor.id
409 },
410 include: [
411 {
412 model: VideoFileModel,
413 required: false,
414 include: [ buildVideoInclude() ]
415 },
416 {
417 model: VideoStreamingPlaylistModel,
418 required: false,
419 include: [ buildVideoInclude() ]
420 }
421 ]
370 } 422 }
371 423
372 return VideoRedundancyModel.findAll(query) 424 return VideoRedundancyModel.findAll(query)
@@ -403,11 +455,32 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
403 })) 455 }))
404 } 456 }
405 457
458 getVideo () {
459 if (this.VideoFile) return this.VideoFile.Video
460
461 return this.VideoStreamingPlaylist.Video
462 }
463
406 isOwned () { 464 isOwned () {
407 return !!this.strategy 465 return !!this.strategy
408 } 466 }
409 467
410 toActivityPubObject (): CacheFileObject { 468 toActivityPubObject (): CacheFileObject {
469 if (this.VideoStreamingPlaylist) {
470 return {
471 id: this.url,
472 type: 'CacheFile' as 'CacheFile',
473 object: this.VideoStreamingPlaylist.Video.url,
474 expires: this.expiresOn.toISOString(),
475 url: {
476 type: 'Link',
477 mimeType: 'application/x-mpegURL',
478 mediaType: 'application/x-mpegURL',
479 href: this.fileUrl
480 }
481 }
482 }
483
411 return { 484 return {
412 id: this.url, 485 id: this.url,
413 type: 'CacheFile' as 'CacheFile', 486 type: 'CacheFile' as 'CacheFile',
@@ -431,7 +504,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
431 504
432 const notIn = Sequelize.literal( 505 const notIn = Sequelize.literal(
433 '(' + 506 '(' +
434 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id}` + 507 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
435 ')' 508 ')'
436 ) 509 )
437 510
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 1f1b76c1e..7d1e371b9 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -62,7 +62,7 @@ export class VideoFileModel extends Model<VideoFileModel> {
62 extname: string 62 extname: string
63 63
64 @AllowNull(false) 64 @AllowNull(false)
65 @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash')) 65 @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash'))
66 @Column 66 @Column
67 infoHash: string 67 infoHash: string
68 68
@@ -86,14 +86,14 @@ export class VideoFileModel extends Model<VideoFileModel> {
86 86
87 @HasMany(() => VideoRedundancyModel, { 87 @HasMany(() => VideoRedundancyModel, {
88 foreignKey: { 88 foreignKey: {
89 allowNull: false 89 allowNull: true
90 }, 90 },
91 onDelete: 'CASCADE', 91 onDelete: 'CASCADE',
92 hooks: true 92 hooks: true
93 }) 93 })
94 RedundancyVideos: VideoRedundancyModel[] 94 RedundancyVideos: VideoRedundancyModel[]
95 95
96 static isInfohashExists (infoHash: string) { 96 static doesInfohashExist (infoHash: string) {
97 const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' 97 const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
98 const options = { 98 const options = {
99 type: Sequelize.QueryTypes.SELECT, 99 type: Sequelize.QueryTypes.SELECT,
diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts
index de0747f22..e49dbee30 100644
--- a/server/models/video/video-format-utils.ts
+++ b/server/models/video/video-format-utils.ts
@@ -1,7 +1,12 @@
1import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' 1import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos'
2import { VideoModel } from './video' 2import { VideoModel } from './video'
3import { VideoFileModel } from './video-file' 3import { VideoFileModel } from './video-file'
4import { ActivityUrlObject, VideoTorrentObject } from '../../../shared/models/activitypub/objects' 4import {
5 ActivityPlaylistInfohashesObject,
6 ActivityPlaylistSegmentHashesObject,
7 ActivityUrlObject,
8 VideoTorrentObject
9} from '../../../shared/models/activitypub/objects'
5import { CONFIG, MIMETYPES, THUMBNAILS_SIZE } from '../../initializers' 10import { CONFIG, MIMETYPES, THUMBNAILS_SIZE } from '../../initializers'
6import { VideoCaptionModel } from './video-caption' 11import { VideoCaptionModel } from './video-caption'
7import { 12import {
@@ -11,6 +16,8 @@ import {
11 getVideoSharesActivityPubUrl 16 getVideoSharesActivityPubUrl
12} from '../../lib/activitypub' 17} from '../../lib/activitypub'
13import { isArray } from '../../helpers/custom-validators/misc' 18import { isArray } from '../../helpers/custom-validators/misc'
19import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model'
20import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
14 21
15export type VideoFormattingJSONOptions = { 22export type VideoFormattingJSONOptions = {
16 completeDescription?: boolean 23 completeDescription?: boolean
@@ -120,7 +127,12 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails {
120 } 127 }
121 }) 128 })
122 129
130 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
131
123 const tags = video.Tags ? video.Tags.map(t => t.name) : [] 132 const tags = video.Tags ? video.Tags.map(t => t.name) : []
133
134 const streamingPlaylists = streamingPlaylistsModelToFormattedJSON(video, video.VideoStreamingPlaylists)
135
124 const detailsJson = { 136 const detailsJson = {
125 support: video.support, 137 support: video.support,
126 descriptionPath: video.getDescriptionAPIPath(), 138 descriptionPath: video.getDescriptionAPIPath(),
@@ -133,7 +145,11 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails {
133 id: video.state, 145 id: video.state,
134 label: VideoModel.getStateLabel(video.state) 146 label: VideoModel.getStateLabel(video.state)
135 }, 147 },
136 files: [] 148
149 trackerUrls: video.getTrackerUrls(baseUrlHttp, baseUrlWs),
150
151 files: [],
152 streamingPlaylists
137 } 153 }
138 154
139 // Format and sort video files 155 // Format and sort video files
@@ -142,6 +158,25 @@ function videoModelToFormattedDetailsJSON (video: VideoModel): VideoDetails {
142 return Object.assign(formattedJson, detailsJson) 158 return Object.assign(formattedJson, detailsJson)
143} 159}
144 160
161function streamingPlaylistsModelToFormattedJSON (video: VideoModel, playlists: VideoStreamingPlaylistModel[]): VideoStreamingPlaylist[] {
162 if (isArray(playlists) === false) return []
163
164 return playlists
165 .map(playlist => {
166 const redundancies = isArray(playlist.RedundancyVideos)
167 ? playlist.RedundancyVideos.map(r => ({ baseUrl: r.fileUrl }))
168 : []
169
170 return {
171 id: playlist.id,
172 type: playlist.type,
173 playlistUrl: playlist.playlistUrl,
174 segmentsSha256Url: playlist.segmentsSha256Url,
175 redundancies
176 } as VideoStreamingPlaylist
177 })
178}
179
145function videoFilesModelToFormattedJSON (video: VideoModel, videoFiles: VideoFileModel[]): VideoFile[] { 180function videoFilesModelToFormattedJSON (video: VideoModel, videoFiles: VideoFileModel[]): VideoFile[] {
146 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() 181 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
147 182
@@ -232,6 +267,28 @@ function videoModelToActivityPubObject (video: VideoModel): VideoTorrentObject {
232 }) 267 })
233 } 268 }
234 269
270 for (const playlist of (video.VideoStreamingPlaylists || [])) {
271 let tag: (ActivityPlaylistSegmentHashesObject | ActivityPlaylistInfohashesObject)[]
272
273 tag = playlist.p2pMediaLoaderInfohashes
274 .map(i => ({ type: 'Infohash' as 'Infohash', name: i }))
275 tag.push({
276 type: 'Link',
277 name: 'sha256',
278 mimeType: 'application/json' as 'application/json',
279 mediaType: 'application/json' as 'application/json',
280 href: playlist.segmentsSha256Url
281 })
282
283 url.push({
284 type: 'Link',
285 mimeType: 'application/x-mpegURL' as 'application/x-mpegURL',
286 mediaType: 'application/x-mpegURL' as 'application/x-mpegURL',
287 href: playlist.playlistUrl,
288 tag
289 })
290 }
291
235 // Add video url too 292 // Add video url too
236 url.push({ 293 url.push({
237 type: 'Link', 294 type: 'Link',
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts
new file mode 100644
index 000000000..bce537781
--- /dev/null
+++ b/server/models/video/video-streaming-playlist.ts
@@ -0,0 +1,154 @@
1import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
2import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
3import { throwIfNotValid } from '../utils'
4import { VideoModel } from './video'
5import * as Sequelize from 'sequelize'
6import { VideoRedundancyModel } from '../redundancy/video-redundancy'
7import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
8import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
9import { CONSTRAINTS_FIELDS, STATIC_PATHS } from '../../initializers'
10import { VideoFileModel } from './video-file'
11import { join } from 'path'
12import { sha1 } from '../../helpers/core-utils'
13import { isArrayOf } from '../../helpers/custom-validators/misc'
14
15@Table({
16 tableName: 'videoStreamingPlaylist',
17 indexes: [
18 {
19 fields: [ 'videoId' ]
20 },
21 {
22 fields: [ 'videoId', 'type' ],
23 unique: true
24 },
25 {
26 fields: [ 'p2pMediaLoaderInfohashes' ],
27 using: 'gin'
28 }
29 ]
30})
31export class VideoStreamingPlaylistModel extends Model<VideoStreamingPlaylistModel> {
32 @CreatedAt
33 createdAt: Date
34
35 @UpdatedAt
36 updatedAt: Date
37
38 @AllowNull(false)
39 @Column
40 type: VideoStreamingPlaylistType
41
42 @AllowNull(false)
43 @Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url'))
44 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max))
45 playlistUrl: string
46
47 @AllowNull(false)
48 @Is('VideoStreamingPlaylistInfoHashes', value => throwIfNotValid(value, v => isArrayOf(v, isVideoFileInfoHashValid), 'info hashes'))
49 @Column(DataType.ARRAY(DataType.STRING))
50 p2pMediaLoaderInfohashes: string[]
51
52 @AllowNull(false)
53 @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url'))
54 @Column
55 segmentsSha256Url: string
56
57 @ForeignKey(() => VideoModel)
58 @Column
59 videoId: number
60
61 @BelongsTo(() => VideoModel, {
62 foreignKey: {
63 allowNull: false
64 },
65 onDelete: 'CASCADE'
66 })
67 Video: VideoModel
68
69 @HasMany(() => VideoRedundancyModel, {
70 foreignKey: {
71 allowNull: false
72 },
73 onDelete: 'CASCADE',
74 hooks: true
75 })
76 RedundancyVideos: VideoRedundancyModel[]
77
78 static doesInfohashExist (infoHash: string) {
79 const query = 'SELECT 1 FROM "videoStreamingPlaylist" WHERE $infoHash = ANY("p2pMediaLoaderInfohashes") LIMIT 1'
80 const options = {
81 type: Sequelize.QueryTypes.SELECT,
82 bind: { infoHash },
83 raw: true
84 }
85
86 return VideoModel.sequelize.query(query, options)
87 .then(results => {
88 return results.length === 1
89 })
90 }
91
92 static buildP2PMediaLoaderInfoHashes (playlistUrl: string, videoFiles: VideoFileModel[]) {
93 const hashes: string[] = []
94
95 // https://github.com/Novage/p2p-media-loader/blob/master/p2p-media-loader-core/lib/p2p-media-manager.ts#L97
96 for (let i = 0; i < videoFiles.length; i++) {
97 hashes.push(sha1(`1${playlistUrl}+V${i}`))
98 }
99
100 return hashes
101 }
102
103 static loadWithVideo (id: number) {
104 const options = {
105 include: [
106 {
107 model: VideoModel.unscoped(),
108 required: true
109 }
110 ]
111 }
112
113 return VideoStreamingPlaylistModel.findById(id, options)
114 }
115
116 static getHlsPlaylistFilename (resolution: number) {
117 return resolution + '.m3u8'
118 }
119
120 static getMasterHlsPlaylistFilename () {
121 return 'master.m3u8'
122 }
123
124 static getHlsSha256SegmentsFilename () {
125 return 'segments-sha256.json'
126 }
127
128 static getHlsMasterPlaylistStaticPath (videoUUID: string) {
129 return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
130 }
131
132 static getHlsPlaylistStaticPath (videoUUID: string, resolution: number) {
133 return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution))
134 }
135
136 static getHlsSha256SegmentsStaticPath (videoUUID: string) {
137 return join(STATIC_PATHS.PLAYLISTS.HLS, videoUUID, VideoStreamingPlaylistModel.getHlsSha256SegmentsFilename())
138 }
139
140 getStringType () {
141 if (this.type === VideoStreamingPlaylistType.HLS) return 'hls'
142
143 return 'unknown'
144 }
145
146 getVideoRedundancyUrl (baseUrlHttp: string) {
147 return baseUrlHttp + STATIC_PATHS.REDUNDANCY + this.getStringType() + '/' + this.Video.uuid
148 }
149
150 hasSameUniqueKeysThan (other: VideoStreamingPlaylistModel) {
151 return this.type === other.type &&
152 this.videoId === other.videoId
153 }
154}
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 80a6c7832..702260772 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -52,7 +52,7 @@ import {
52 ACTIVITY_PUB, 52 ACTIVITY_PUB,
53 API_VERSION, 53 API_VERSION,
54 CONFIG, 54 CONFIG,
55 CONSTRAINTS_FIELDS, 55 CONSTRAINTS_FIELDS, HLS_PLAYLIST_DIRECTORY, HLS_REDUNDANCY_DIRECTORY,
56 PREVIEWS_SIZE, 56 PREVIEWS_SIZE,
57 REMOTE_SCHEME, 57 REMOTE_SCHEME,
58 STATIC_DOWNLOAD_PATHS, 58 STATIC_DOWNLOAD_PATHS,
@@ -95,6 +95,7 @@ import * as validator from 'validator'
95import { UserVideoHistoryModel } from '../account/user-video-history' 95import { UserVideoHistoryModel } from '../account/user-video-history'
96import { UserModel } from '../account/user' 96import { UserModel } from '../account/user'
97import { VideoImportModel } from './video-import' 97import { VideoImportModel } from './video-import'
98import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
98 99
99// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation 100// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
100const indexes: Sequelize.DefineIndexesOptions[] = [ 101const indexes: Sequelize.DefineIndexesOptions[] = [
@@ -159,7 +160,9 @@ export enum ScopeNames {
159 WITH_FILES = 'WITH_FILES', 160 WITH_FILES = 'WITH_FILES',
160 WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE', 161 WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE',
161 WITH_BLACKLISTED = 'WITH_BLACKLISTED', 162 WITH_BLACKLISTED = 'WITH_BLACKLISTED',
162 WITH_USER_HISTORY = 'WITH_USER_HISTORY' 163 WITH_USER_HISTORY = 'WITH_USER_HISTORY',
164 WITH_STREAMING_PLAYLISTS = 'WITH_STREAMING_PLAYLISTS',
165 WITH_USER_ID = 'WITH_USER_ID'
163} 166}
164 167
165type ForAPIOptions = { 168type ForAPIOptions = {
@@ -463,6 +466,22 @@ type AvailableForListIDsOptions = {
463 466
464 return query 467 return query
465 }, 468 },
469 [ ScopeNames.WITH_USER_ID ]: {
470 include: [
471 {
472 attributes: [ 'accountId' ],
473 model: () => VideoChannelModel.unscoped(),
474 required: true,
475 include: [
476 {
477 attributes: [ 'userId' ],
478 model: () => AccountModel.unscoped(),
479 required: true
480 }
481 ]
482 }
483 ]
484 },
466 [ ScopeNames.WITH_ACCOUNT_DETAILS ]: { 485 [ ScopeNames.WITH_ACCOUNT_DETAILS ]: {
467 include: [ 486 include: [
468 { 487 {
@@ -527,22 +546,55 @@ type AvailableForListIDsOptions = {
527 } 546 }
528 ] 547 ]
529 }, 548 },
530 [ ScopeNames.WITH_FILES ]: { 549 [ ScopeNames.WITH_FILES ]: (withRedundancies = false) => {
531 include: [ 550 let subInclude: any[] = []
532 { 551
533 model: () => VideoFileModel.unscoped(), 552 if (withRedundancies === true) {
534 // FIXME: typings 553 subInclude = [
535 [ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join 554 {
536 required: false, 555 attributes: [ 'fileUrl' ],
537 include: [ 556 model: VideoRedundancyModel.unscoped(),
538 { 557 required: false
539 attributes: [ 'fileUrl' ], 558 }
540 model: () => VideoRedundancyModel.unscoped(), 559 ]
541 required: false 560 }
542 } 561
543 ] 562 return {
544 } 563 include: [
545 ] 564 {
565 model: VideoFileModel.unscoped(),
566 // FIXME: typings
567 [ 'separate' as any ]: true, // We may have multiple files, having multiple redundancies so let's separate this join
568 required: false,
569 include: subInclude
570 }
571 ]
572 }
573 },
574 [ ScopeNames.WITH_STREAMING_PLAYLISTS ]: (withRedundancies = false) => {
575 let subInclude: any[] = []
576
577 if (withRedundancies === true) {
578 subInclude = [
579 {
580 attributes: [ 'fileUrl' ],
581 model: VideoRedundancyModel.unscoped(),
582 required: false
583 }
584 ]
585 }
586
587 return {
588 include: [
589 {
590 model: VideoStreamingPlaylistModel.unscoped(),
591 // FIXME: typings
592 [ 'separate' as any ]: true, // We may have multiple streaming playlists, having multiple redundancies so let's separate this join
593 required: false,
594 include: subInclude
595 }
596 ]
597 }
546 }, 598 },
547 [ ScopeNames.WITH_SCHEDULED_UPDATE ]: { 599 [ ScopeNames.WITH_SCHEDULED_UPDATE ]: {
548 include: [ 600 include: [
@@ -722,6 +774,16 @@ export class VideoModel extends Model<VideoModel> {
722 }) 774 })
723 VideoFiles: VideoFileModel[] 775 VideoFiles: VideoFileModel[]
724 776
777 @HasMany(() => VideoStreamingPlaylistModel, {
778 foreignKey: {
779 name: 'videoId',
780 allowNull: false
781 },
782 hooks: true,
783 onDelete: 'cascade'
784 })
785 VideoStreamingPlaylists: VideoStreamingPlaylistModel[]
786
725 @HasMany(() => VideoShareModel, { 787 @HasMany(() => VideoShareModel, {
726 foreignKey: { 788 foreignKey: {
727 name: 'videoId', 789 name: 'videoId',
@@ -847,6 +909,9 @@ export class VideoModel extends Model<VideoModel> {
847 tasks.push(instance.removeFile(file)) 909 tasks.push(instance.removeFile(file))
848 tasks.push(instance.removeTorrent(file)) 910 tasks.push(instance.removeTorrent(file))
849 }) 911 })
912
913 // Remove playlists file
914 tasks.push(instance.removeStreamingPlaylist())
850 } 915 }
851 916
852 // Do not wait video deletion because we could be in a transaction 917 // Do not wait video deletion because we could be in a transaction
@@ -858,10 +923,6 @@ export class VideoModel extends Model<VideoModel> {
858 return undefined 923 return undefined
859 } 924 }
860 925
861 static list () {
862 return VideoModel.scope(ScopeNames.WITH_FILES).findAll()
863 }
864
865 static listLocal () { 926 static listLocal () {
866 const query = { 927 const query = {
867 where: { 928 where: {
@@ -869,7 +930,7 @@ export class VideoModel extends Model<VideoModel> {
869 } 930 }
870 } 931 }
871 932
872 return VideoModel.scope(ScopeNames.WITH_FILES).findAll(query) 933 return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ]).findAll(query)
873 } 934 }
874 935
875 static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) { 936 static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) {
@@ -1200,6 +1261,16 @@ export class VideoModel extends Model<VideoModel> {
1200 return VideoModel.findOne(options) 1261 return VideoModel.findOne(options)
1201 } 1262 }
1202 1263
1264 static loadWithRights (id: number | string, t?: Sequelize.Transaction) {
1265 const where = VideoModel.buildWhereIdOrUUID(id)
1266 const options = {
1267 where,
1268 transaction: t
1269 }
1270
1271 return VideoModel.scope([ ScopeNames.WITH_BLACKLISTED, ScopeNames.WITH_USER_ID ]).findOne(options)
1272 }
1273
1203 static loadOnlyId (id: number | string, t?: Sequelize.Transaction) { 1274 static loadOnlyId (id: number | string, t?: Sequelize.Transaction) {
1204 const where = VideoModel.buildWhereIdOrUUID(id) 1275 const where = VideoModel.buildWhereIdOrUUID(id)
1205 1276
@@ -1212,8 +1283,8 @@ export class VideoModel extends Model<VideoModel> {
1212 return VideoModel.findOne(options) 1283 return VideoModel.findOne(options)
1213 } 1284 }
1214 1285
1215 static loadWithFile (id: number, t?: Sequelize.Transaction, logging?: boolean) { 1286 static loadWithFiles (id: number, t?: Sequelize.Transaction, logging?: boolean) {
1216 return VideoModel.scope(ScopeNames.WITH_FILES) 1287 return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS ])
1217 .findById(id, { transaction: t, logging }) 1288 .findById(id, { transaction: t, logging })
1218 } 1289 }
1219 1290
@@ -1224,9 +1295,7 @@ export class VideoModel extends Model<VideoModel> {
1224 } 1295 }
1225 } 1296 }
1226 1297
1227 return VideoModel 1298 return VideoModel.findOne(options)
1228 .scope([ ScopeNames.WITH_FILES ])
1229 .findOne(options)
1230 } 1299 }
1231 1300
1232 static loadByUrl (url: string, transaction?: Sequelize.Transaction) { 1301 static loadByUrl (url: string, transaction?: Sequelize.Transaction) {
@@ -1248,7 +1317,11 @@ export class VideoModel extends Model<VideoModel> {
1248 transaction 1317 transaction
1249 } 1318 }
1250 1319
1251 return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) 1320 return VideoModel.scope([
1321 ScopeNames.WITH_ACCOUNT_DETAILS,
1322 ScopeNames.WITH_FILES,
1323 ScopeNames.WITH_STREAMING_PLAYLISTS
1324 ]).findOne(query)
1252 } 1325 }
1253 1326
1254 static loadAndPopulateAccountAndServerAndTags (id: number | string, t?: Sequelize.Transaction, userId?: number) { 1327 static loadAndPopulateAccountAndServerAndTags (id: number | string, t?: Sequelize.Transaction, userId?: number) {
@@ -1263,9 +1336,37 @@ export class VideoModel extends Model<VideoModel> {
1263 const scopes = [ 1336 const scopes = [
1264 ScopeNames.WITH_TAGS, 1337 ScopeNames.WITH_TAGS,
1265 ScopeNames.WITH_BLACKLISTED, 1338 ScopeNames.WITH_BLACKLISTED,
1339 ScopeNames.WITH_ACCOUNT_DETAILS,
1340 ScopeNames.WITH_SCHEDULED_UPDATE,
1266 ScopeNames.WITH_FILES, 1341 ScopeNames.WITH_FILES,
1342 ScopeNames.WITH_STREAMING_PLAYLISTS
1343 ]
1344
1345 if (userId) {
1346 scopes.push({ method: [ ScopeNames.WITH_USER_HISTORY, userId ] } as any) // FIXME: typings
1347 }
1348
1349 return VideoModel
1350 .scope(scopes)
1351 .findOne(options)
1352 }
1353
1354 static loadForGetAPI (id: number | string, t?: Sequelize.Transaction, userId?: number) {
1355 const where = VideoModel.buildWhereIdOrUUID(id)
1356
1357 const options = {
1358 order: [ [ 'Tags', 'name', 'ASC' ] ],
1359 where,
1360 transaction: t
1361 }
1362
1363 const scopes = [
1364 ScopeNames.WITH_TAGS,
1365 ScopeNames.WITH_BLACKLISTED,
1267 ScopeNames.WITH_ACCOUNT_DETAILS, 1366 ScopeNames.WITH_ACCOUNT_DETAILS,
1268 ScopeNames.WITH_SCHEDULED_UPDATE 1367 ScopeNames.WITH_SCHEDULED_UPDATE,
1368 { method: [ ScopeNames.WITH_FILES, true ] } as any, // FIXME: typings
1369 { method: [ ScopeNames.WITH_STREAMING_PLAYLISTS, true ] } as any // FIXME: typings
1269 ] 1370 ]
1270 1371
1271 if (userId) { 1372 if (userId) {
@@ -1612,6 +1713,14 @@ export class VideoModel extends Model<VideoModel> {
1612 .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) 1713 .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err }))
1613 } 1714 }
1614 1715
1716 removeStreamingPlaylist (isRedundancy = false) {
1717 const baseDir = isRedundancy ? HLS_REDUNDANCY_DIRECTORY : HLS_PLAYLIST_DIRECTORY
1718
1719 const filePath = join(baseDir, this.uuid)
1720 return remove(filePath)
1721 .catch(err => logger.warn('Cannot delete playlist directory %s.', filePath, { err }))
1722 }
1723
1615 isOutdated () { 1724 isOutdated () {
1616 if (this.isOwned()) return false 1725 if (this.isOwned()) return false
1617 1726
@@ -1646,7 +1755,7 @@ export class VideoModel extends Model<VideoModel> {
1646 1755
1647 generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) { 1756 generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) {
1648 const xs = this.getTorrentUrl(videoFile, baseUrlHttp) 1757 const xs = this.getTorrentUrl(videoFile, baseUrlHttp)
1649 const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] 1758 const announce = this.getTrackerUrls(baseUrlHttp, baseUrlWs)
1650 let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ] 1759 let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ]
1651 1760
1652 const redundancies = videoFile.RedundancyVideos 1761 const redundancies = videoFile.RedundancyVideos
@@ -1663,6 +1772,10 @@ export class VideoModel extends Model<VideoModel> {
1663 return magnetUtil.encode(magnetHash) 1772 return magnetUtil.encode(magnetHash)
1664 } 1773 }
1665 1774
1775 getTrackerUrls (baseUrlHttp: string, baseUrlWs: string) {
1776 return [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ]
1777 }
1778
1666 getThumbnailUrl (baseUrlHttp: string) { 1779 getThumbnailUrl (baseUrlHttp: string) {
1667 return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName() 1780 return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName()
1668 } 1781 }
@@ -1686,4 +1799,8 @@ export class VideoModel extends Model<VideoModel> {
1686 getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { 1799 getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
1687 return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile) 1800 return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile)
1688 } 1801 }
1802
1803 getBandwidthBits (videoFile: VideoFileModel) {
1804 return Math.ceil((videoFile.size * 8) / this.duration)
1805 }
1689} 1806}
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts
index 4038ecbf0..07de2b5a5 100644
--- a/server/tests/api/check-params/config.ts
+++ b/server/tests/api/check-params/config.ts
@@ -65,6 +65,9 @@ describe('Test config API validators', function () {
65 '480p': true, 65 '480p': true,
66 '720p': false, 66 '720p': false,
67 '1080p': false 67 '1080p': false
68 },
69 hls: {
70 enabled: false
68 } 71 }
69 }, 72 },
70 import: { 73 import: {
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index 9d3ce8153..5b99309fb 100644
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -17,7 +17,7 @@ import {
17 viewVideo, 17 viewVideo,
18 wait, 18 wait,
19 waitUntilLog, 19 waitUntilLog,
20 checkVideoFilesWereRemoved, removeVideo, getVideoWithToken 20 checkVideoFilesWereRemoved, removeVideo, getVideoWithToken, reRunServer
21} from '../../../../shared/utils' 21} from '../../../../shared/utils'
22import { waitJobs } from '../../../../shared/utils/server/jobs' 22import { waitJobs } from '../../../../shared/utils/server/jobs'
23 23
@@ -48,6 +48,11 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe
48 48
49async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { 49async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) {
50 const config = { 50 const config = {
51 transcoding: {
52 hls: {
53 enabled: true
54 }
55 },
51 redundancy: { 56 redundancy: {
52 videos: { 57 videos: {
53 check_interval: '5 seconds', 58 check_interval: '5 seconds',
@@ -85,7 +90,7 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams:
85 await waitJobs(servers) 90 await waitJobs(servers)
86} 91}
87 92
88async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) { 93async function check1WebSeed (videoUUID?: string) {
89 if (!videoUUID) videoUUID = video1Server2UUID 94 if (!videoUUID) videoUUID = video1Server2UUID
90 95
91 const webseeds = [ 96 const webseeds = [
@@ -93,47 +98,17 @@ async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: str
93 ] 98 ]
94 99
95 for (const server of servers) { 100 for (const server of servers) {
96 { 101 // With token to avoid issues with video follow constraints
97 // With token to avoid issues with video follow constraints 102 const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
98 const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
99 103
100 const video: VideoDetails = res.body 104 const video: VideoDetails = res.body
101 for (const f of video.files) { 105 for (const f of video.files) {
102 checkMagnetWebseeds(f, webseeds, server) 106 checkMagnetWebseeds(f, webseeds, server)
103 }
104 } 107 }
105 } 108 }
106} 109}
107 110
108async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { 111async function check2Webseeds (videoUUID?: string) {
109 const res = await getStats(servers[0].url)
110 const data: ServerStats = res.body
111
112 expect(data.videosRedundancy).to.have.lengthOf(1)
113 const stat = data.videosRedundancy[0]
114
115 expect(stat.strategy).to.equal(strategy)
116 expect(stat.totalSize).to.equal(204800)
117 expect(stat.totalUsed).to.be.at.least(1).and.below(204801)
118 expect(stat.totalVideoFiles).to.equal(4)
119 expect(stat.totalVideos).to.equal(1)
120}
121
122async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
123 const res = await getStats(servers[0].url)
124 const data: ServerStats = res.body
125
126 expect(data.videosRedundancy).to.have.lengthOf(1)
127
128 const stat = data.videosRedundancy[0]
129 expect(stat.strategy).to.equal(strategy)
130 expect(stat.totalSize).to.equal(204800)
131 expect(stat.totalUsed).to.equal(0)
132 expect(stat.totalVideoFiles).to.equal(0)
133 expect(stat.totalVideos).to.equal(0)
134}
135
136async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) {
137 if (!videoUUID) videoUUID = video1Server2UUID 112 if (!videoUUID) videoUUID = video1Server2UUID
138 113
139 const webseeds = [ 114 const webseeds = [
@@ -158,7 +133,7 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st
158 await makeGetRequest({ 133 await makeGetRequest({
159 url: servers[1].url, 134 url: servers[1].url,
160 statusCodeExpected: 200, 135 statusCodeExpected: 200,
161 path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`, 136 path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`,
162 contentType: null 137 contentType: null
163 }) 138 })
164 } 139 }
@@ -174,6 +149,81 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st
174 } 149 }
175} 150}
176 151
152async function check0PlaylistRedundancies (videoUUID?: string) {
153 if (!videoUUID) videoUUID = video1Server2UUID
154
155 for (const server of servers) {
156 // With token to avoid issues with video follow constraints
157 const res = await getVideoWithToken(server.url, server.accessToken, videoUUID)
158 const video: VideoDetails = res.body
159
160 expect(video.streamingPlaylists).to.be.an('array')
161 expect(video.streamingPlaylists).to.have.lengthOf(1)
162 expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0)
163 }
164}
165
166async function check1PlaylistRedundancies (videoUUID?: string) {
167 if (!videoUUID) videoUUID = video1Server2UUID
168
169 for (const server of servers) {
170 const res = await getVideo(server.url, videoUUID)
171 const video: VideoDetails = res.body
172
173 expect(video.streamingPlaylists).to.have.lengthOf(1)
174 expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1)
175
176 const redundancy = video.streamingPlaylists[0].redundancies[0]
177
178 expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
179 }
180
181 await makeGetRequest({
182 url: servers[0].url,
183 statusCodeExpected: 200,
184 path: `/static/redundancy/hls/${videoUUID}/360_000.ts`,
185 contentType: null
186 })
187
188 for (const directory of [ 'test1/redundancy/hls', 'test2/playlists/hls' ]) {
189 const files = await readdir(join(root(), directory, videoUUID))
190 expect(files).to.have.length.at.least(4)
191
192 for (const resolution of [ 240, 360, 480, 720 ]) {
193 expect(files.find(f => f === `${resolution}_000.ts`)).to.not.be.undefined
194 expect(files.find(f => f === `${resolution}_001.ts`)).to.not.be.undefined
195 }
196 }
197}
198
199async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) {
200 const res = await getStats(servers[0].url)
201 const data: ServerStats = res.body
202
203 expect(data.videosRedundancy).to.have.lengthOf(1)
204 const stat = data.videosRedundancy[0]
205
206 expect(stat.strategy).to.equal(strategy)
207 expect(stat.totalSize).to.equal(204800)
208 expect(stat.totalUsed).to.be.at.least(1).and.below(204801)
209 expect(stat.totalVideoFiles).to.equal(4)
210 expect(stat.totalVideos).to.equal(1)
211}
212
213async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
214 const res = await getStats(servers[0].url)
215 const data: ServerStats = res.body
216
217 expect(data.videosRedundancy).to.have.lengthOf(1)
218
219 const stat = data.videosRedundancy[0]
220 expect(stat.strategy).to.equal(strategy)
221 expect(stat.totalSize).to.equal(204800)
222 expect(stat.totalUsed).to.equal(0)
223 expect(stat.totalVideoFiles).to.equal(0)
224 expect(stat.totalVideos).to.equal(0)
225}
226
177async function enableRedundancyOnServer1 () { 227async function enableRedundancyOnServer1 () {
178 await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) 228 await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
179 229
@@ -220,7 +270,8 @@ describe('Test videos redundancy', function () {
220 }) 270 })
221 271
222 it('Should have 1 webseed on the first video', async function () { 272 it('Should have 1 webseed on the first video', async function () {
223 await check1WebSeed(strategy) 273 await check1WebSeed()
274 await check0PlaylistRedundancies()
224 await checkStatsWith1Webseed(strategy) 275 await checkStatsWith1Webseed(strategy)
225 }) 276 })
226 277
@@ -229,27 +280,29 @@ describe('Test videos redundancy', function () {
229 }) 280 })
230 281
231 it('Should have 2 webseeds on the first video', async function () { 282 it('Should have 2 webseeds on the first video', async function () {
232 this.timeout(40000) 283 this.timeout(80000)
233 284
234 await waitJobs(servers) 285 await waitJobs(servers)
235 await waitUntilLog(servers[0], 'Duplicated ', 4) 286 await waitUntilLog(servers[0], 'Duplicated ', 5)
236 await waitJobs(servers) 287 await waitJobs(servers)
237 288
238 await check2Webseeds(strategy) 289 await check2Webseeds()
290 await check1PlaylistRedundancies()
239 await checkStatsWith2Webseed(strategy) 291 await checkStatsWith2Webseed(strategy)
240 }) 292 })
241 293
242 it('Should undo redundancy on server 1 and remove duplicated videos', async function () { 294 it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
243 this.timeout(40000) 295 this.timeout(80000)
244 296
245 await disableRedundancyOnServer1() 297 await disableRedundancyOnServer1()
246 298
247 await waitJobs(servers) 299 await waitJobs(servers)
248 await wait(5000) 300 await wait(5000)
249 301
250 await check1WebSeed(strategy) 302 await check1WebSeed()
303 await check0PlaylistRedundancies()
251 304
252 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) 305 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos', join('playlists', 'hls') ])
253 }) 306 })
254 307
255 after(function () { 308 after(function () {
@@ -267,7 +320,8 @@ describe('Test videos redundancy', function () {
267 }) 320 })
268 321
269 it('Should have 1 webseed on the first video', async function () { 322 it('Should have 1 webseed on the first video', async function () {
270 await check1WebSeed(strategy) 323 await check1WebSeed()
324 await check0PlaylistRedundancies()
271 await checkStatsWith1Webseed(strategy) 325 await checkStatsWith1Webseed(strategy)
272 }) 326 })
273 327
@@ -276,25 +330,27 @@ describe('Test videos redundancy', function () {
276 }) 330 })
277 331
278 it('Should have 2 webseeds on the first video', async function () { 332 it('Should have 2 webseeds on the first video', async function () {
279 this.timeout(40000) 333 this.timeout(80000)
280 334
281 await waitJobs(servers) 335 await waitJobs(servers)
282 await waitUntilLog(servers[0], 'Duplicated ', 4) 336 await waitUntilLog(servers[0], 'Duplicated ', 5)
283 await waitJobs(servers) 337 await waitJobs(servers)
284 338
285 await check2Webseeds(strategy) 339 await check2Webseeds()
340 await check1PlaylistRedundancies()
286 await checkStatsWith2Webseed(strategy) 341 await checkStatsWith2Webseed(strategy)
287 }) 342 })
288 343
289 it('Should unfollow on server 1 and remove duplicated videos', async function () { 344 it('Should unfollow on server 1 and remove duplicated videos', async function () {
290 this.timeout(40000) 345 this.timeout(80000)
291 346
292 await unfollow(servers[0].url, servers[0].accessToken, servers[1]) 347 await unfollow(servers[0].url, servers[0].accessToken, servers[1])
293 348
294 await waitJobs(servers) 349 await waitJobs(servers)
295 await wait(5000) 350 await wait(5000)
296 351
297 await check1WebSeed(strategy) 352 await check1WebSeed()
353 await check0PlaylistRedundancies()
298 354
299 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) 355 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
300 }) 356 })
@@ -314,7 +370,8 @@ describe('Test videos redundancy', function () {
314 }) 370 })
315 371
316 it('Should have 1 webseed on the first video', async function () { 372 it('Should have 1 webseed on the first video', async function () {
317 await check1WebSeed(strategy) 373 await check1WebSeed()
374 await check0PlaylistRedundancies()
318 await checkStatsWith1Webseed(strategy) 375 await checkStatsWith1Webseed(strategy)
319 }) 376 })
320 377
@@ -323,18 +380,19 @@ describe('Test videos redundancy', function () {
323 }) 380 })
324 381
325 it('Should still have 1 webseed on the first video', async function () { 382 it('Should still have 1 webseed on the first video', async function () {
326 this.timeout(40000) 383 this.timeout(80000)
327 384
328 await waitJobs(servers) 385 await waitJobs(servers)
329 await wait(15000) 386 await wait(15000)
330 await waitJobs(servers) 387 await waitJobs(servers)
331 388
332 await check1WebSeed(strategy) 389 await check1WebSeed()
390 await check0PlaylistRedundancies()
333 await checkStatsWith1Webseed(strategy) 391 await checkStatsWith1Webseed(strategy)
334 }) 392 })
335 393
336 it('Should view 2 times the first video to have > min_views config', async function () { 394 it('Should view 2 times the first video to have > min_views config', async function () {
337 this.timeout(40000) 395 this.timeout(80000)
338 396
339 await viewVideo(servers[ 0 ].url, video1Server2UUID) 397 await viewVideo(servers[ 0 ].url, video1Server2UUID)
340 await viewVideo(servers[ 2 ].url, video1Server2UUID) 398 await viewVideo(servers[ 2 ].url, video1Server2UUID)
@@ -344,13 +402,14 @@ describe('Test videos redundancy', function () {
344 }) 402 })
345 403
346 it('Should have 2 webseeds on the first video', async function () { 404 it('Should have 2 webseeds on the first video', async function () {
347 this.timeout(40000) 405 this.timeout(80000)
348 406
349 await waitJobs(servers) 407 await waitJobs(servers)
350 await waitUntilLog(servers[0], 'Duplicated ', 4) 408 await waitUntilLog(servers[0], 'Duplicated ', 5)
351 await waitJobs(servers) 409 await waitJobs(servers)
352 410
353 await check2Webseeds(strategy) 411 await check2Webseeds()
412 await check1PlaylistRedundancies()
354 await checkStatsWith2Webseed(strategy) 413 await checkStatsWith2Webseed(strategy)
355 }) 414 })
356 415
@@ -405,7 +464,7 @@ describe('Test videos redundancy', function () {
405 }) 464 })
406 465
407 it('Should still have 2 webseeds after 10 seconds', async function () { 466 it('Should still have 2 webseeds after 10 seconds', async function () {
408 this.timeout(40000) 467 this.timeout(80000)
409 468
410 await wait(10000) 469 await wait(10000)
411 470
@@ -420,7 +479,7 @@ describe('Test videos redundancy', function () {
420 }) 479 })
421 480
422 it('Should stop server 1 and expire video redundancy', async function () { 481 it('Should stop server 1 and expire video redundancy', async function () {
423 this.timeout(40000) 482 this.timeout(80000)
424 483
425 killallServers([ servers[0] ]) 484 killallServers([ servers[0] ])
426 485
@@ -446,10 +505,11 @@ describe('Test videos redundancy', function () {
446 await enableRedundancyOnServer1() 505 await enableRedundancyOnServer1()
447 506
448 await waitJobs(servers) 507 await waitJobs(servers)
449 await waitUntilLog(servers[0], 'Duplicated ', 4) 508 await waitUntilLog(servers[0], 'Duplicated ', 5)
450 await waitJobs(servers) 509 await waitJobs(servers)
451 510
452 await check2Webseeds(strategy) 511 await check2Webseeds()
512 await check1PlaylistRedundancies()
453 await checkStatsWith2Webseed(strategy) 513 await checkStatsWith2Webseed(strategy)
454 514
455 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) 515 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
@@ -467,8 +527,10 @@ describe('Test videos redundancy', function () {
467 await wait(1000) 527 await wait(1000)
468 528
469 try { 529 try {
470 await check1WebSeed(strategy, video1Server2UUID) 530 await check1WebSeed(video1Server2UUID)
471 await check2Webseeds(strategy, video2Server2UUID) 531 await check0PlaylistRedundancies(video1Server2UUID)
532 await check2Webseeds(video2Server2UUID)
533 await check1PlaylistRedundancies(video2Server2UUID)
472 534
473 checked = true 535 checked = true
474 } catch { 536 } catch {
@@ -477,6 +539,26 @@ describe('Test videos redundancy', function () {
477 } 539 }
478 }) 540 })
479 541
542 it('Should disable strategy and remove redundancies', async function () {
543 this.timeout(80000)
544
545 await waitJobs(servers)
546
547 killallServers([ servers[ 0 ] ])
548 await reRunServer(servers[ 0 ], {
549 redundancy: {
550 videos: {
551 check_interval: '1 second',
552 strategies: []
553 }
554 }
555 })
556
557 await waitJobs(servers)
558
559 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ join('redundancy', 'hls') ])
560 })
561
480 after(function () { 562 after(function () {
481 return cleanServers() 563 return cleanServers()
482 }) 564 })
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts
index bebfc7398..0dfe6e4fe 100644
--- a/server/tests/api/server/config.ts
+++ b/server/tests/api/server/config.ts
@@ -57,6 +57,8 @@ function checkInitialConfig (data: CustomConfig) {
57 expect(data.transcoding.resolutions['480p']).to.be.true 57 expect(data.transcoding.resolutions['480p']).to.be.true
58 expect(data.transcoding.resolutions['720p']).to.be.true 58 expect(data.transcoding.resolutions['720p']).to.be.true
59 expect(data.transcoding.resolutions['1080p']).to.be.true 59 expect(data.transcoding.resolutions['1080p']).to.be.true
60 expect(data.transcoding.hls.enabled).to.be.true
61
60 expect(data.import.videos.http.enabled).to.be.true 62 expect(data.import.videos.http.enabled).to.be.true
61 expect(data.import.videos.torrent.enabled).to.be.true 63 expect(data.import.videos.torrent.enabled).to.be.true
62} 64}
@@ -95,6 +97,7 @@ function checkUpdatedConfig (data: CustomConfig) {
95 expect(data.transcoding.resolutions['480p']).to.be.true 97 expect(data.transcoding.resolutions['480p']).to.be.true
96 expect(data.transcoding.resolutions['720p']).to.be.false 98 expect(data.transcoding.resolutions['720p']).to.be.false
97 expect(data.transcoding.resolutions['1080p']).to.be.false 99 expect(data.transcoding.resolutions['1080p']).to.be.false
100 expect(data.transcoding.hls.enabled).to.be.false
98 101
99 expect(data.import.videos.http.enabled).to.be.false 102 expect(data.import.videos.http.enabled).to.be.false
100 expect(data.import.videos.torrent.enabled).to.be.false 103 expect(data.import.videos.torrent.enabled).to.be.false
@@ -205,6 +208,9 @@ describe('Test config', function () {
205 '480p': true, 208 '480p': true,
206 '720p': false, 209 '720p': false,
207 '1080p': false 210 '1080p': false
211 },
212 hls: {
213 enabled: false
208 } 214 }
209 }, 215 },
210 import: { 216 import: {
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts
index 97f467aae..a501a80b2 100644
--- a/server/tests/api/videos/index.ts
+++ b/server/tests/api/videos/index.ts
@@ -8,6 +8,7 @@ import './video-change-ownership'
8import './video-channels' 8import './video-channels'
9import './video-comments' 9import './video-comments'
10import './video-description' 10import './video-description'
11import './video-hls'
11import './video-imports' 12import './video-imports'
12import './video-nsfw' 13import './video-nsfw'
13import './video-privacy' 14import './video-privacy'
diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts
new file mode 100644
index 000000000..71d863b12
--- /dev/null
+++ b/server/tests/api/videos/video-hls.ts
@@ -0,0 +1,145 @@
1/* tslint:disable:no-unused-expression */
2
3import * as chai from 'chai'
4import 'mocha'
5import {
6 checkDirectoryIsEmpty,
7 checkTmpIsEmpty,
8 doubleFollow,
9 flushAndRunMultipleServers,
10 flushTests,
11 getPlaylist,
12 getSegment,
13 getSegmentSha256,
14 getVideo,
15 killallServers,
16 removeVideo,
17 ServerInfo,
18 setAccessTokensToServers,
19 updateVideo,
20 uploadVideo,
21 waitJobs
22} from '../../../../shared/utils'
23import { VideoDetails } from '../../../../shared/models/videos'
24import { VideoStreamingPlaylistType } from '../../../../shared/models/videos/video-streaming-playlist.type'
25import { sha256 } from '../../../helpers/core-utils'
26import { join } from 'path'
27
28const expect = chai.expect
29
30async function checkHlsPlaylist (servers: ServerInfo[], videoUUID: string) {
31 const resolutions = [ 240, 360, 480, 720 ]
32
33 for (const server of servers) {
34 const res = await getVideo(server.url, videoUUID)
35 const videoDetails: VideoDetails = res.body
36
37 expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
38
39 const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
40 expect(hlsPlaylist).to.not.be.undefined
41
42 {
43 const res2 = await getPlaylist(hlsPlaylist.playlistUrl)
44
45 const masterPlaylist = res2.text
46
47 expect(masterPlaylist).to.contain('#EXT-X-STREAM-INF:BANDWIDTH=55472,RESOLUTION=640x360,FRAME-RATE=25')
48
49 for (const resolution of resolutions) {
50 expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
51 }
52 }
53
54 {
55 for (const resolution of resolutions) {
56 const res2 = await getPlaylist(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}.m3u8`)
57
58 const subPlaylist = res2.text
59 expect(subPlaylist).to.contain(resolution + '_000.ts')
60 }
61 }
62
63 {
64 for (const resolution of resolutions) {
65
66 const res2 = await getSegment(`http://localhost:9001/static/playlists/hls/${videoUUID}/${resolution}_000.ts`)
67
68 const resSha = await getSegmentSha256(hlsPlaylist.segmentsSha256Url)
69
70 const sha256Server = resSha.body[ resolution + '_000.ts' ]
71 expect(sha256(res2.body)).to.equal(sha256Server)
72 }
73 }
74 }
75}
76
77describe('Test HLS videos', function () {
78 let servers: ServerInfo[] = []
79 let videoUUID = ''
80
81 before(async function () {
82 this.timeout(120000)
83
84 servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: true, hls: { enabled: true } } })
85
86 // Get the access tokens
87 await setAccessTokensToServers(servers)
88
89 // Server 1 and server 2 follow each other
90 await doubleFollow(servers[0], servers[1])
91 })
92
93 it('Should upload a video and transcode it to HLS', async function () {
94 this.timeout(120000)
95
96 {
97 const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, { name: 'video 1', fixture: 'video_short.webm' })
98 videoUUID = res.body.video.uuid
99 }
100
101 await waitJobs(servers)
102
103 await checkHlsPlaylist(servers, videoUUID)
104 })
105
106 it('Should update the video', async function () {
107 await updateVideo(servers[0].url, servers[0].accessToken, videoUUID, { name: 'video 1 updated' })
108
109 await waitJobs(servers)
110
111 await checkHlsPlaylist(servers, videoUUID)
112 })
113
114 it('Should delete the video', async function () {
115 await removeVideo(servers[0].url, servers[0].accessToken, videoUUID)
116
117 await waitJobs(servers)
118
119 for (const server of servers) {
120 await getVideo(server.url, videoUUID, 404)
121 }
122 })
123
124 it('Should have the playlists/segment deleted from the disk', async function () {
125 for (const server of servers) {
126 await checkDirectoryIsEmpty(server, 'videos')
127 await checkDirectoryIsEmpty(server, join('playlists', 'hls'))
128 }
129 })
130
131 it('Should have an empty tmp directory', async function () {
132 for (const server of servers) {
133 await checkTmpIsEmpty(server)
134 }
135 })
136
137 after(async function () {
138 killallServers(servers)
139
140 // Keep the logs if the test failed
141 if (this['ok']) {
142 await flushTests()
143 }
144 })
145})
diff --git a/server/tests/cli/update-host.ts b/server/tests/cli/update-host.ts
index 811ea6a9f..d38bb4331 100644
--- a/server/tests/cli/update-host.ts
+++ b/server/tests/cli/update-host.ts
@@ -86,6 +86,13 @@ describe('Test update host scripts', function () {
86 const { body } = await makeActivityPubGetRequest(server.url, '/videos/watch/' + video.uuid) 86 const { body } = await makeActivityPubGetRequest(server.url, '/videos/watch/' + video.uuid)
87 87
88 expect(body.id).to.equal('http://localhost:9002/videos/watch/' + video.uuid) 88 expect(body.id).to.equal('http://localhost:9002/videos/watch/' + video.uuid)
89
90 const res = await getVideo(server.url, video.uuid)
91 const videoDetails: VideoDetails = res.body
92
93 expect(videoDetails.trackerUrls[0]).to.include(server.host)
94 expect(videoDetails.streamingPlaylists[0].playlistUrl).to.include(server.host)
95 expect(videoDetails.streamingPlaylists[0].segmentsSha256Url).to.include(server.host)
89 } 96 }
90 }) 97 })
91 98
@@ -100,7 +107,7 @@ describe('Test update host scripts', function () {
100 } 107 }
101 }) 108 })
102 109
103 it('Should have update accounts url', async function () { 110 it('Should have updated accounts url', async function () {
104 const res = await getAccountsList(server.url) 111 const res = await getAccountsList(server.url)
105 expect(res.body.total).to.equal(3) 112 expect(res.body.total).to.equal(3)
106 113
@@ -112,7 +119,7 @@ describe('Test update host scripts', function () {
112 } 119 }
113 }) 120 })
114 121
115 it('Should update torrent hosts', async function () { 122 it('Should have updated torrent hosts', async function () {
116 this.timeout(30000) 123 this.timeout(30000)
117 124
118 const res = await getVideosList(server.url) 125 const res = await getVideosList(server.url)
diff --git a/shared/models/activitypub/objects/cache-file-object.ts b/shared/models/activitypub/objects/cache-file-object.ts
index 0a5125f5b..4b0a3a724 100644
--- a/shared/models/activitypub/objects/cache-file-object.ts
+++ b/shared/models/activitypub/objects/cache-file-object.ts
@@ -1,9 +1,9 @@
1import { ActivityVideoUrlObject } from './common-objects' 1import { ActivityVideoUrlObject, ActivityPlaylistUrlObject } from './common-objects'
2 2
3export interface CacheFileObject { 3export interface CacheFileObject {
4 id: string 4 id: string
5 type: 'CacheFile', 5 type: 'CacheFile',
6 object: string 6 object: string
7 expires: string 7 expires: string
8 url: ActivityVideoUrlObject 8 url: ActivityVideoUrlObject | ActivityPlaylistUrlObject
9} 9}
diff --git a/shared/models/activitypub/objects/common-objects.ts b/shared/models/activitypub/objects/common-objects.ts
index 118a4f43d..8c89810d6 100644
--- a/shared/models/activitypub/objects/common-objects.ts
+++ b/shared/models/activitypub/objects/common-objects.ts
@@ -28,25 +28,47 @@ export type ActivityVideoUrlObject = {
28 fps: number 28 fps: number
29} 29}
30 30
31export type ActivityUrlObject = 31export type ActivityPlaylistSegmentHashesObject = {
32 ActivityVideoUrlObject 32 type: 'Link'
33 | 33 name: 'sha256'
34 { 34 // TODO: remove mimeType (backward compatibility, introduced in v1.1.0)
35 type: 'Link' 35 mimeType?: 'application/json'
36 // TODO: remove mimeType (backward compatibility, introduced in v1.1.0) 36 mediaType: 'application/json'
37 mimeType?: 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet' 37 href: string
38 mediaType: 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet' 38}
39 href: string 39
40 height: number 40export type ActivityPlaylistInfohashesObject = {
41 } 41 type: 'Infohash'
42 | 42 name: string
43 { 43}
44 type: 'Link' 44
45 // TODO: remove mimeType (backward compatibility, introduced in v1.1.0) 45export type ActivityPlaylistUrlObject = {
46 mimeType?: 'text/html' 46 type: 'Link'
47 mediaType: 'text/html' 47 // TODO: remove mimeType (backward compatibility, introduced in v1.1.0)
48 href: string 48 mimeType?: 'application/x-mpegURL'
49 } 49 mediaType: 'application/x-mpegURL'
50 href: string
51 tag?: (ActivityPlaylistSegmentHashesObject | ActivityPlaylistInfohashesObject)[]
52}
53
54export type ActivityBitTorrentUrlObject = {
55 type: 'Link'
56 // TODO: remove mimeType (backward compatibility, introduced in v1.1.0)
57 mimeType?: 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet'
58 mediaType: 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet'
59 href: string
60 height: number
61}
62
63export type ActivityHtmlUrlObject = {
64 type: 'Link'
65 // TODO: remove mimeType (backward compatibility, introduced in v1.1.0)
66 mimeType?: 'text/html'
67 mediaType: 'text/html'
68 href: string
69}
70
71export type ActivityUrlObject = ActivityVideoUrlObject | ActivityPlaylistUrlObject | ActivityBitTorrentUrlObject | ActivityHtmlUrlObject
50 72
51export interface ActivityPubAttributedTo { 73export interface ActivityPubAttributedTo {
52 type: 'Group' | 'Person' 74 type: 'Group' | 'Person'
diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts
index 7a3eaa33f..b42ff90c6 100644
--- a/shared/models/server/custom-config.model.ts
+++ b/shared/models/server/custom-config.model.ts
@@ -61,6 +61,9 @@ export interface CustomConfig {
61 '720p': boolean 61 '720p': boolean
62 '1080p': boolean 62 '1080p': boolean
63 } 63 }
64 hls: {
65 enabled: boolean
66 }
64 } 67 }
65 68
66 import: { 69 import: {
diff --git a/shared/models/server/server-config.model.ts b/shared/models/server/server-config.model.ts
index f4245ed4d..baafed31f 100644
--- a/shared/models/server/server-config.model.ts
+++ b/shared/models/server/server-config.model.ts
@@ -25,11 +25,15 @@ export interface ServerConfig {
25 25
26 signup: { 26 signup: {
27 allowed: boolean, 27 allowed: boolean,
28 allowedForCurrentIP: boolean, 28 allowedForCurrentIP: boolean
29 requiresEmailVerification: boolean 29 requiresEmailVerification: boolean
30 } 30 }
31 31
32 transcoding: { 32 transcoding: {
33 hls: {
34 enabled: boolean
35 }
36
33 enabledResolutions: number[] 37 enabledResolutions: number[]
34 } 38 }
35 39
@@ -48,7 +52,7 @@ export interface ServerConfig {
48 file: { 52 file: {
49 size: { 53 size: {
50 max: number 54 max: number
51 }, 55 }
52 extensions: string[] 56 extensions: string[]
53 } 57 }
54 } 58 }
diff --git a/shared/models/videos/video-streaming-playlist.model.ts b/shared/models/videos/video-streaming-playlist.model.ts
new file mode 100644
index 000000000..17f8fe865
--- /dev/null
+++ b/shared/models/videos/video-streaming-playlist.model.ts
@@ -0,0 +1,12 @@
1import { VideoStreamingPlaylistType } from './video-streaming-playlist.type'
2
3export class VideoStreamingPlaylist {
4 id: number
5 type: VideoStreamingPlaylistType
6 playlistUrl: string
7 segmentsSha256Url: string
8
9 redundancies: {
10 baseUrl: string
11 }[]
12}
diff --git a/shared/models/videos/video-streaming-playlist.type.ts b/shared/models/videos/video-streaming-playlist.type.ts
new file mode 100644
index 000000000..3b403f295
--- /dev/null
+++ b/shared/models/videos/video-streaming-playlist.type.ts
@@ -0,0 +1,3 @@
1export enum VideoStreamingPlaylistType {
2 HLS = 1
3}
diff --git a/shared/models/videos/video.model.ts b/shared/models/videos/video.model.ts
index 022876a0b..803db8255 100644
--- a/shared/models/videos/video.model.ts
+++ b/shared/models/videos/video.model.ts
@@ -5,6 +5,7 @@ import { VideoChannel } from './channel/video-channel.model'
5import { VideoPrivacy } from './video-privacy.enum' 5import { VideoPrivacy } from './video-privacy.enum'
6import { VideoScheduleUpdate } from './video-schedule-update.model' 6import { VideoScheduleUpdate } from './video-schedule-update.model'
7import { VideoConstant } from './video-constant.model' 7import { VideoConstant } from './video-constant.model'
8import { VideoStreamingPlaylist } from './video-streaming-playlist.model'
8 9
9export interface VideoFile { 10export interface VideoFile {
10 magnetUri: string 11 magnetUri: string
@@ -86,4 +87,8 @@ export interface VideoDetails extends Video {
86 // Not optional in details (unlike in Video) 87 // Not optional in details (unlike in Video)
87 waitTranscoding: boolean 88 waitTranscoding: boolean
88 state: VideoConstant<VideoState> 89 state: VideoConstant<VideoState>
90
91 trackerUrls: string[]
92
93 streamingPlaylists: VideoStreamingPlaylist[]
89} 94}
diff --git a/shared/utils/index.ts b/shared/utils/index.ts
index e08bbfd2a..156901372 100644
--- a/shared/utils/index.ts
+++ b/shared/utils/index.ts
@@ -17,6 +17,8 @@ export * from './users/users'
17export * from './videos/video-abuses' 17export * from './videos/video-abuses'
18export * from './videos/video-blacklist' 18export * from './videos/video-blacklist'
19export * from './videos/video-channels' 19export * from './videos/video-channels'
20export * from './videos/video-comments'
21export * from './videos/video-playlists'
20export * from './videos/videos' 22export * from './videos/videos'
21export * from './videos/video-change-ownership' 23export * from './videos/video-change-ownership'
22export * from './feeds/feeds' 24export * from './feeds/feeds'
diff --git a/shared/utils/requests/requests.ts b/shared/utils/requests/requests.ts
index 77e9f6164..fc687c701 100644
--- a/shared/utils/requests/requests.ts
+++ b/shared/utils/requests/requests.ts
@@ -1,10 +1,17 @@
1import * as request from 'supertest' 1import * as request from 'supertest'
2import { buildAbsoluteFixturePath, root } from '../miscs/miscs' 2import { buildAbsoluteFixturePath, root } from '../miscs/miscs'
3import { isAbsolute, join } from 'path' 3import { isAbsolute, join } from 'path'
4import { parse } from 'url'
5
6function makeRawRequest (url: string, statusCodeExpected?: number) {
7 const { host, protocol, pathname } = parse(url)
8
9 return makeGetRequest({ url: `${protocol}//${host}`, path: pathname, statusCodeExpected })
10}
4 11
5function makeGetRequest (options: { 12function makeGetRequest (options: {
6 url: string, 13 url: string,
7 path: string, 14 path?: string,
8 query?: any, 15 query?: any,
9 token?: string, 16 token?: string,
10 statusCodeExpected?: number, 17 statusCodeExpected?: number,
@@ -13,8 +20,7 @@ function makeGetRequest (options: {
13 if (!options.statusCodeExpected) options.statusCodeExpected = 400 20 if (!options.statusCodeExpected) options.statusCodeExpected = 400
14 if (options.contentType === undefined) options.contentType = 'application/json' 21 if (options.contentType === undefined) options.contentType = 'application/json'
15 22
16 const req = request(options.url) 23 const req = request(options.url).get(options.path)
17 .get(options.path)
18 24
19 if (options.contentType) req.set('Accept', options.contentType) 25 if (options.contentType) req.set('Accept', options.contentType)
20 if (options.token) req.set('Authorization', 'Bearer ' + options.token) 26 if (options.token) req.set('Authorization', 'Bearer ' + options.token)
@@ -164,5 +170,6 @@ export {
164 makePostBodyRequest, 170 makePostBodyRequest,
165 makePutBodyRequest, 171 makePutBodyRequest,
166 makeDeleteRequest, 172 makeDeleteRequest,
173 makeRawRequest,
167 updateAvatarRequest 174 updateAvatarRequest
168} 175}
diff --git a/shared/utils/server/config.ts b/shared/utils/server/config.ts
index 0c5512bab..29c24cff9 100644
--- a/shared/utils/server/config.ts
+++ b/shared/utils/server/config.ts
@@ -97,6 +97,9 @@ function updateCustomSubConfig (url: string, token: string, newConfig: any) {
97 '480p': true, 97 '480p': true,
98 '720p': false, 98 '720p': false,
99 '1080p': false 99 '1080p': false
100 },
101 hls: {
102 enabled: false
100 } 103 }
101 }, 104 },
102 import: { 105 import: {
diff --git a/shared/utils/server/servers.ts b/shared/utils/server/servers.ts
index cb57e0a69..bde7dd5c2 100644
--- a/shared/utils/server/servers.ts
+++ b/shared/utils/server/servers.ts
@@ -166,9 +166,13 @@ async function reRunServer (server: ServerInfo, configOverride?: any) {
166} 166}
167 167
168async function checkTmpIsEmpty (server: ServerInfo) { 168async function checkTmpIsEmpty (server: ServerInfo) {
169 return checkDirectoryIsEmpty(server, 'tmp')
170}
171
172async function checkDirectoryIsEmpty (server: ServerInfo, directory: string) {
169 const testDirectory = 'test' + server.serverNumber 173 const testDirectory = 'test' + server.serverNumber
170 174
171 const directoryPath = join(root(), testDirectory, 'tmp') 175 const directoryPath = join(root(), testDirectory, directory)
172 176
173 const directoryExists = existsSync(directoryPath) 177 const directoryExists = existsSync(directoryPath)
174 expect(directoryExists).to.be.true 178 expect(directoryExists).to.be.true
@@ -199,6 +203,7 @@ async function waitUntilLog (server: ServerInfo, str: string, count = 1) {
199// --------------------------------------------------------------------------- 203// ---------------------------------------------------------------------------
200 204
201export { 205export {
206 checkDirectoryIsEmpty,
202 checkTmpIsEmpty, 207 checkTmpIsEmpty,
203 ServerInfo, 208 ServerInfo,
204 flushAndRunMultipleServers, 209 flushAndRunMultipleServers,
diff --git a/shared/utils/videos/video-playlists.ts b/shared/utils/videos/video-playlists.ts
new file mode 100644
index 000000000..9a0710ca6
--- /dev/null
+++ b/shared/utils/videos/video-playlists.ts
@@ -0,0 +1,21 @@
1import { makeRawRequest } from '../requests/requests'
2
3function getPlaylist (url: string, statusCodeExpected = 200) {
4 return makeRawRequest(url, statusCodeExpected)
5}
6
7function getSegment (url: string, statusCodeExpected = 200) {
8 return makeRawRequest(url, statusCodeExpected)
9}
10
11function getSegmentSha256 (url: string, statusCodeExpected = 200) {
12 return makeRawRequest(url, statusCodeExpected)
13}
14
15// ---------------------------------------------------------------------------
16
17export {
18 getPlaylist,
19 getSegment,
20 getSegmentSha256
21}
diff --git a/shared/utils/videos/videos.ts b/shared/utils/videos/videos.ts
index 0cf6e7c4f..b5b33e038 100644
--- a/shared/utils/videos/videos.ts
+++ b/shared/utils/videos/videos.ts
@@ -271,7 +271,16 @@ function removeVideo (url: string, token: string, id: number | string, expectedS
271async function checkVideoFilesWereRemoved ( 271async function checkVideoFilesWereRemoved (
272 videoUUID: string, 272 videoUUID: string,
273 serverNumber: number, 273 serverNumber: number,
274 directories = [ 'redundancy', 'videos', 'thumbnails', 'torrents', 'previews', 'captions' ] 274 directories = [
275 'redundancy',
276 'videos',
277 'thumbnails',
278 'torrents',
279 'previews',
280 'captions',
281 join('playlists', 'hls'),
282 join('redundancy', 'hls')
283 ]
275) { 284) {
276 const testDirectory = 'test' + serverNumber 285 const testDirectory = 'test' + serverNumber
277 286
@@ -279,7 +288,7 @@ async function checkVideoFilesWereRemoved (
279 const directoryPath = join(root(), testDirectory, directory) 288 const directoryPath = join(root(), testDirectory, directory)
280 289
281 const directoryExists = existsSync(directoryPath) 290 const directoryExists = existsSync(directoryPath)
282 expect(directoryExists).to.be.true 291 if (!directoryExists) continue
283 292
284 const files = await readdir(directoryPath) 293 const files = await readdir(directoryPath)
285 for (const file of files) { 294 for (const file of files) {
diff --git a/yarn.lock b/yarn.lock
index 1e759af1b..47c0646e4 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2,6 +2,14 @@
2# yarn lockfile v1 2# yarn lockfile v1
3 3
4 4
5"@babel/polyfill@^7.2.5":
6 version "7.2.5"
7 resolved "https://registry.yarnpkg.com/@babel/polyfill/-/polyfill-7.2.5.tgz#6c54b964f71ad27edddc567d065e57e87ed7fa7d"
8 integrity sha512-8Y/t3MWThtMLYr0YNC/Q76tqN1w30+b0uQMeFUYauG2UGTR19zyUtFrAzT23zNtBxPp+LbE5E/nwV/q/r3y6ug==
9 dependencies:
10 core-js "^2.5.7"
11 regenerator-runtime "^0.12.0"
12
5"@iamstarkov/listr-update-renderer@0.4.1": 13"@iamstarkov/listr-update-renderer@0.4.1":
6 version "0.4.1" 14 version "0.4.1"
7 resolved "https://registry.yarnpkg.com/@iamstarkov/listr-update-renderer/-/listr-update-renderer-0.4.1.tgz#d7c48092a2dcf90fd672b6c8b458649cb350c77e" 15 resolved "https://registry.yarnpkg.com/@iamstarkov/listr-update-renderer/-/listr-update-renderer-0.4.1.tgz#d7c48092a2dcf90fd672b6c8b458649cb350c77e"
@@ -3585,6 +3593,17 @@ hide-powered-by@1.0.0:
3585 resolved "https://registry.yarnpkg.com/hide-powered-by/-/hide-powered-by-1.0.0.tgz#4a85ad65881f62857fc70af7174a1184dccce32b" 3593 resolved "https://registry.yarnpkg.com/hide-powered-by/-/hide-powered-by-1.0.0.tgz#4a85ad65881f62857fc70af7174a1184dccce32b"
3586 integrity sha1-SoWtZYgfYoV/xwr3F0oRhNzM4ys= 3594 integrity sha1-SoWtZYgfYoV/xwr3F0oRhNzM4ys=
3587 3595
3596"hlsdownloader@https://github.com/Chocobozzz/hlsdownloader#build":
3597 version "0.0.0-semantic-release"
3598 resolved "https://github.com/Chocobozzz/hlsdownloader#e19f9d803dcfe7ec25fd734b4743184f19a9b0cc"
3599 dependencies:
3600 "@babel/polyfill" "^7.2.5"
3601 async "^2.6.1"
3602 minimist "^1.2.0"
3603 mkdirp "^0.5.1"
3604 request "^2.88.0"
3605 request-promise "^4.2.2"
3606
3588hosted-git-info@^2.1.4, hosted-git-info@^2.6.0, hosted-git-info@^2.7.1: 3607hosted-git-info@^2.1.4, hosted-git-info@^2.6.0, hosted-git-info@^2.7.1:
3589 version "2.7.1" 3608 version "2.7.1"
3590 resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.7.1.tgz#97f236977bd6e125408930ff6de3eec6281ec047" 3609 resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.7.1.tgz#97f236977bd6e125408930ff6de3eec6281ec047"
@@ -4851,7 +4870,7 @@ lodash@=3.10.1:
4851 resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.10.1.tgz#5bf45e8e49ba4189e17d482789dfd15bd140b7b6" 4870 resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.10.1.tgz#5bf45e8e49ba4189e17d482789dfd15bd140b7b6"
4852 integrity sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y= 4871 integrity sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y=
4853 4872
4854lodash@^4.0.0, lodash@^4.17.1, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.5, lodash@^4.3.0, lodash@^4.8.2, lodash@~4.17.10: 4873lodash@^4.0.0, lodash@^4.13.1, lodash@^4.17.1, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.5, lodash@^4.3.0, lodash@^4.8.2, lodash@~4.17.10:
4855 version "4.17.11" 4874 version "4.17.11"
4856 resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d" 4875 resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d"
4857 integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg== 4876 integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==
@@ -6632,6 +6651,11 @@ psl@^1.1.24:
6632 resolved "https://registry.yarnpkg.com/psl/-/psl-1.1.29.tgz#60f580d360170bb722a797cc704411e6da850c67" 6651 resolved "https://registry.yarnpkg.com/psl/-/psl-1.1.29.tgz#60f580d360170bb722a797cc704411e6da850c67"
6633 integrity sha512-AeUmQ0oLN02flVHXWh9sSJF7mcdFq0ppid/JkErufc3hGIV/AMa8Fo9VgDo/cT2jFdOWoFvHp90qqBH54W+gjQ== 6652 integrity sha512-AeUmQ0oLN02flVHXWh9sSJF7mcdFq0ppid/JkErufc3hGIV/AMa8Fo9VgDo/cT2jFdOWoFvHp90qqBH54W+gjQ==
6634 6653
6654psl@^1.1.28:
6655 version "1.1.31"
6656 resolved "https://registry.yarnpkg.com/psl/-/psl-1.1.31.tgz#e9aa86d0101b5b105cbe93ac6b784cd547276184"
6657 integrity sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw==
6658
6635pstree.remy@^1.1.2: 6659pstree.remy@^1.1.2:
6636 version "1.1.2" 6660 version "1.1.2"
6637 resolved "https://registry.yarnpkg.com/pstree.remy/-/pstree.remy-1.1.2.tgz#4448bbeb4b2af1fed242afc8dc7416a6f504951a" 6661 resolved "https://registry.yarnpkg.com/pstree.remy/-/pstree.remy-1.1.2.tgz#4448bbeb4b2af1fed242afc8dc7416a6f504951a"
@@ -6675,7 +6699,7 @@ punycode@^1.4.1:
6675 resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" 6699 resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e"
6676 integrity sha1-wNWmOycYgArY4esPpSachN1BhF4= 6700 integrity sha1-wNWmOycYgArY4esPpSachN1BhF4=
6677 6701
6678punycode@^2.1.0: 6702punycode@^2.1.0, punycode@^2.1.1:
6679 version "2.1.1" 6703 version "2.1.1"
6680 resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" 6704 resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec"
6681 integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== 6705 integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==
@@ -6958,6 +6982,11 @@ reflect-metadata@^0.1.12:
6958 resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.12.tgz#311bf0c6b63cd782f228a81abe146a2bfa9c56f2" 6982 resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.12.tgz#311bf0c6b63cd782f228a81abe146a2bfa9c56f2"
6959 integrity sha512-n+IyV+nGz3+0q3/Yf1ra12KpCyi001bi4XFxSjbiWWjfqb52iTTtpGXmCCAOWWIAn9KEuFZKGqBERHmrtScZ3A== 6983 integrity sha512-n+IyV+nGz3+0q3/Yf1ra12KpCyi001bi4XFxSjbiWWjfqb52iTTtpGXmCCAOWWIAn9KEuFZKGqBERHmrtScZ3A==
6960 6984
6985regenerator-runtime@^0.12.0:
6986 version "0.12.1"
6987 resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.12.1.tgz#fa1a71544764c036f8c49b13a08b2594c9f8a0de"
6988 integrity sha512-odxIc1/vDlo4iZcfXqRYFj0vpXFNoGdKMAUieAlFYO6m/nl5e9KR/beGf41z4a1FI+aQgtjhuaSlDxQ0hmkrHg==
6989
6961regex-not@^1.0.0, regex-not@^1.0.2: 6990regex-not@^1.0.0, regex-not@^1.0.2:
6962 version "1.0.2" 6991 version "1.0.2"
6963 resolved "https://registry.yarnpkg.com/regex-not/-/regex-not-1.0.2.tgz#1f4ece27e00b0b65e0247a6810e6a85d83a5752c" 6992 resolved "https://registry.yarnpkg.com/regex-not/-/regex-not-1.0.2.tgz#1f4ece27e00b0b65e0247a6810e6a85d83a5752c"
@@ -7007,6 +7036,23 @@ repeat-string@^1.6.1:
7007 resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" 7036 resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637"
7008 integrity sha1-jcrkcOHIirwtYA//Sndihtp15jc= 7037 integrity sha1-jcrkcOHIirwtYA//Sndihtp15jc=
7009 7038
7039request-promise-core@1.1.1:
7040 version "1.1.1"
7041 resolved "https://registry.yarnpkg.com/request-promise-core/-/request-promise-core-1.1.1.tgz#3eee00b2c5aa83239cfb04c5700da36f81cd08b6"
7042 integrity sha1-Pu4AssWqgyOc+wTFcA2jb4HNCLY=
7043 dependencies:
7044 lodash "^4.13.1"
7045
7046request-promise@^4.2.2:
7047 version "4.2.2"
7048 resolved "https://registry.yarnpkg.com/request-promise/-/request-promise-4.2.2.tgz#d1ea46d654a6ee4f8ee6a4fea1018c22911904b4"
7049 integrity sha1-0epG1lSm7k+O5qT+oQGMIpEZBLQ=
7050 dependencies:
7051 bluebird "^3.5.0"
7052 request-promise-core "1.1.1"
7053 stealthy-require "^1.1.0"
7054 tough-cookie ">=2.3.3"
7055
7010request@^2.74.0, request@^2.81.0, request@^2.83.0, request@^2.87.0, request@^2.88.0: 7056request@^2.74.0, request@^2.81.0, request@^2.83.0, request@^2.87.0, request@^2.88.0:
7011 version "2.88.0" 7057 version "2.88.0"
7012 resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef" 7058 resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef"
@@ -7924,6 +7970,11 @@ statuses@~1.4.0:
7924 resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.4.0.tgz#bb73d446da2796106efcc1b601a253d6c46bd087" 7970 resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.4.0.tgz#bb73d446da2796106efcc1b601a253d6c46bd087"
7925 integrity sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew== 7971 integrity sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew==
7926 7972
7973stealthy-require@^1.1.0:
7974 version "1.1.1"
7975 resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b"
7976 integrity sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=
7977
7927stream-each@^1.1.0: 7978stream-each@^1.1.0:
7928 version "1.2.3" 7979 version "1.2.3"
7929 resolved "https://registry.yarnpkg.com/stream-each/-/stream-each-1.2.3.tgz#ebe27a0c389b04fbcc233642952e10731afa9bae" 7980 resolved "https://registry.yarnpkg.com/stream-each/-/stream-each-1.2.3.tgz#ebe27a0c389b04fbcc233642952e10731afa9bae"
@@ -8416,6 +8467,15 @@ touch@^3.1.0:
8416 dependencies: 8467 dependencies:
8417 nopt "~1.0.10" 8468 nopt "~1.0.10"
8418 8469
8470tough-cookie@>=2.3.3:
8471 version "3.0.1"
8472 resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-3.0.1.tgz#9df4f57e739c26930a018184887f4adb7dca73b2"
8473 integrity sha512-yQyJ0u4pZsv9D4clxO69OEjLWYw+jbgspjTue4lTQZLfV0c5l1VmK2y1JK8E9ahdpltPOaAThPcp5nKPUgSnsg==
8474 dependencies:
8475 ip-regex "^2.1.0"
8476 psl "^1.1.28"
8477 punycode "^2.1.1"
8478
8419tough-cookie@~2.4.3: 8479tough-cookie@~2.4.3:
8420 version "2.4.3" 8480 version "2.4.3"
8421 resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.4.3.tgz#53f36da3f47783b0925afa06ff9f3b165280f781" 8481 resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.4.3.tgz#53f36da3f47783b0925afa06ff9f3b165280f781"