aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/videos.ts141
-rw-r--r--server/lib/hls.ts13
-rw-r--r--server/lib/job-queue/handlers/video-file-import.ts6
-rw-r--r--server/lib/job-queue/handlers/video-import.ts12
-rw-r--r--server/lib/job-queue/handlers/video-transcoding.ts119
-rw-r--r--server/lib/schedulers/update-videos-scheduler.ts8
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts7
-rw-r--r--server/lib/thumbnail.ts3
-rw-r--r--server/lib/video-paths.ts64
-rw-r--r--server/lib/video-transcoding.ts91
-rw-r--r--server/lib/videos.ts11
11 files changed, 314 insertions, 161 deletions
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index c318978fd..d80173e03 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -3,8 +3,10 @@ import * as sequelize from 'sequelize'
3import * as magnetUtil from 'magnet-uri' 3import * as magnetUtil from 'magnet-uri'
4import * as request from 'request' 4import * as request from 'request'
5import { 5import {
6 ActivityHashTagObject,
7 ActivityMagnetUrlObject,
6 ActivityPlaylistSegmentHashesObject, 8 ActivityPlaylistSegmentHashesObject,
7 ActivityPlaylistUrlObject, 9 ActivityPlaylistUrlObject, ActivityTagObject,
8 ActivityUrlObject, 10 ActivityUrlObject,
9 ActivityVideoUrlObject, 11 ActivityVideoUrlObject,
10 VideoState 12 VideoState
@@ -13,7 +15,7 @@ import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
13import { VideoPrivacy } from '../../../shared/models/videos' 15import { VideoPrivacy } from '../../../shared/models/videos'
14import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' 16import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
15import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' 17import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
16import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' 18import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
17import { logger } from '../../helpers/logger' 19import { logger } from '../../helpers/logger'
18import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' 20import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
19import { 21import {
@@ -57,6 +59,7 @@ import {
57 MChannelAccountLight, 59 MChannelAccountLight,
58 MChannelDefault, 60 MChannelDefault,
59 MChannelId, 61 MChannelId,
62 MStreamingPlaylist,
60 MVideo, 63 MVideo,
61 MVideoAccountLight, 64 MVideoAccountLight,
62 MVideoAccountLightBlacklistAllFiles, 65 MVideoAccountLightBlacklistAllFiles,
@@ -330,21 +333,15 @@ async function updateVideoFromAP (options: {
330 await videoUpdated.addAndSaveThumbnail(previewModel, t) 333 await videoUpdated.addAndSaveThumbnail(previewModel, t)
331 334
332 { 335 {
333 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoUpdated, videoObject) 336 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoUpdated, videoObject.url)
334 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) 337 const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
335 338
336 // Remove video files that do not exist anymore 339 // Remove video files that do not exist anymore
337 const destroyTasks = videoUpdated.VideoFiles 340 const destroyTasks = deleteNonExistingModels(videoUpdated.VideoFiles, newVideoFiles, t)
338 .filter(f => !newVideoFiles.find(newFile => newFile.hasSameUniqueKeysThan(f)))
339 .map(f => f.destroy(sequelizeOptions))
340 await Promise.all(destroyTasks) 341 await Promise.all(destroyTasks)
341 342
342 // Update or add other one 343 // Update or add other one
343 const upsertTasks = videoFileAttributes.map(a => { 344 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t))
344 return VideoFileModel.upsert<VideoFileModel>(a, { returning: true, transaction: t })
345 .then(([ file ]) => file)
346 })
347
348 videoUpdated.VideoFiles = await Promise.all(upsertTasks) 345 videoUpdated.VideoFiles = await Promise.all(upsertTasks)
349 } 346 }
350 347
@@ -352,24 +349,39 @@ async function updateVideoFromAP (options: {
352 const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(videoUpdated, videoObject, videoUpdated.VideoFiles) 349 const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(videoUpdated, videoObject, videoUpdated.VideoFiles)
353 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a)) 350 const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
354 351
355 // Remove video files that do not exist anymore 352 // Remove video playlists that do not exist anymore
356 const destroyTasks = videoUpdated.VideoStreamingPlaylists 353 const destroyTasks = deleteNonExistingModels(videoUpdated.VideoStreamingPlaylists, newStreamingPlaylists, t)
357 .filter(f => !newStreamingPlaylists.find(newPlaylist => newPlaylist.hasSameUniqueKeysThan(f)))
358 .map(f => f.destroy(sequelizeOptions))
359 await Promise.all(destroyTasks) 354 await Promise.all(destroyTasks)
360 355
361 // Update or add other one 356 let oldStreamingPlaylistFiles: MVideoFile[] = []
362 const upsertTasks = streamingPlaylistAttributes.map(a => { 357 for (const videoStreamingPlaylist of videoUpdated.VideoStreamingPlaylists) {
363 return VideoStreamingPlaylistModel.upsert<VideoStreamingPlaylistModel>(a, { returning: true, transaction: t }) 358 oldStreamingPlaylistFiles = oldStreamingPlaylistFiles.concat(videoStreamingPlaylist.VideoFiles)
364 .then(([ streamingPlaylist ]) => streamingPlaylist) 359 }
365 }) 360
361 videoUpdated.VideoStreamingPlaylists = []
362
363 for (const playlistAttributes of streamingPlaylistAttributes) {
364 const streamingPlaylistModel = await VideoStreamingPlaylistModel.upsert(playlistAttributes, { returning: true, transaction: t })
365 .then(([ streamingPlaylist ]) => streamingPlaylist)
366 366
367 videoUpdated.VideoStreamingPlaylists = await Promise.all(upsertTasks) 367 const newVideoFiles: MVideoFile[] = videoFileActivityUrlToDBAttributes(streamingPlaylistModel, playlistAttributes.tagAPObject)
368 .map(a => new VideoFileModel(a))
369 const destroyTasks = deleteNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles, t)
370 await Promise.all(destroyTasks)
371
372 // Update or add other one
373 const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t))
374 streamingPlaylistModel.VideoFiles = await Promise.all(upsertTasks)
375
376 videoUpdated.VideoStreamingPlaylists.push(streamingPlaylistModel)
377 }
368 } 378 }
369 379
370 { 380 {
371 // Update Tags 381 // Update Tags
372 const tags = videoObject.tag.map(tag => tag.name) 382 const tags = videoObject.tag
383 .filter(isAPHashTagObject)
384 .map(tag => tag.name)
373 const tagInstances = await TagModel.findOrCreateTags(tags, t) 385 const tagInstances = await TagModel.findOrCreateTags(tags, t)
374 await videoUpdated.$set('Tags', tagInstances, sequelizeOptions) 386 await videoUpdated.$set('Tags', tagInstances, sequelizeOptions)
375 } 387 }
@@ -478,23 +490,27 @@ export {
478 490
479// --------------------------------------------------------------------------- 491// ---------------------------------------------------------------------------
480 492
481function isAPVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { 493function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject {
482 const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) 494 const mimeTypes = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT)
483 495
484 const urlMediaType = url.mediaType || url.mimeType 496 const urlMediaType = url.mediaType
485 return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/') 497 return mimeTypes.indexOf(urlMediaType) !== -1 && urlMediaType.startsWith('video/')
486} 498}
487 499
488function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject { 500function isAPStreamingPlaylistUrlObject (url: ActivityUrlObject): url is ActivityPlaylistUrlObject {
489 const urlMediaType = url.mediaType || url.mimeType 501 return url && url.mediaType === 'application/x-mpegURL'
490
491 return urlMediaType === 'application/x-mpegURL'
492} 502}
493 503
494function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject { 504function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject {
495 const urlMediaType = tag.mediaType || tag.mimeType 505 return tag && tag.name === 'sha256' && tag.type === 'Link' && tag.mediaType === 'application/json'
506}
507
508function isAPMagnetUrlObject (url: any): url is ActivityMagnetUrlObject {
509 return url && url.mediaType === 'application/x-bittorrent;x-scheme-handler/magnet'
510}
496 511
497 return tag.name === 'sha256' && tag.type === 'Link' && urlMediaType === 'application/json' 512function isAPHashTagObject (url: any): url is ActivityHashTagObject {
513 return url && url.type === 'Hashtag'
498} 514}
499 515
500async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAccountLight, waitThumbnail = false) { 516async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAccountLight, waitThumbnail = false) {
@@ -524,21 +540,27 @@ async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAc
524 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(previewModel, t) 540 if (thumbnailModel) await videoCreated.addAndSaveThumbnail(previewModel, t)
525 541
526 // Process files 542 // Process files
527 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject) 543 const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject.url)
528 if (videoFileAttributes.length === 0) {
529 throw new Error('Cannot find valid files for video %s ' + videoObject.url)
530 }
531 544
532 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) 545 const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
533 const videoFiles = await Promise.all(videoFilePromises) 546 const videoFiles = await Promise.all(videoFilePromises)
534 547
535 const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles) 548 const streamingPlaylistsAttributes = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles)
536 const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t })) 549 videoCreated.VideoStreamingPlaylists = []
537 const streamingPlaylists = await Promise.all(playlistPromises) 550
551 for (const playlistAttributes of streamingPlaylistsAttributes) {
552 const playlistModel = await VideoStreamingPlaylistModel.create(playlistAttributes, { transaction: t })
553
554 const playlistFiles = videoFileActivityUrlToDBAttributes(playlistModel, playlistAttributes.tagAPObject)
555 const videoFilePromises = playlistFiles.map(f => VideoFileModel.create(f, { transaction: t }))
556 playlistModel.VideoFiles = await Promise.all(videoFilePromises)
557
558 videoCreated.VideoStreamingPlaylists.push(playlistModel)
559 }
538 560
539 // Process tags 561 // Process tags
540 const tags = videoObject.tag 562 const tags = videoObject.tag
541 .filter(t => t.type === 'Hashtag') 563 .filter(isAPHashTagObject)
542 .map(t => t.name) 564 .map(t => t.name)
543 const tagInstances = await TagModel.findOrCreateTags(tags, t) 565 const tagInstances = await TagModel.findOrCreateTags(tags, t)
544 await videoCreated.$set('Tags', tagInstances, sequelizeOptions) 566 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
@@ -550,7 +572,6 @@ async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAc
550 await Promise.all(videoCaptionsPromises) 572 await Promise.all(videoCaptionsPromises)
551 573
552 videoCreated.VideoFiles = videoFiles 574 videoCreated.VideoFiles = videoFiles
553 videoCreated.VideoStreamingPlaylists = streamingPlaylists
554 videoCreated.Tags = tagInstances 575 videoCreated.Tags = tagInstances
555 576
556 const autoBlacklisted = await autoBlacklistVideoIfNeeded({ 577 const autoBlacklisted = await autoBlacklistVideoIfNeeded({
@@ -628,20 +649,19 @@ async function videoActivityObjectToDBAttributes (videoChannel: MChannelId, vide
628 } 649 }
629} 650}
630 651
631function videoFileActivityUrlToDBAttributes (video: MVideo, videoObject: VideoTorrentObject) { 652function videoFileActivityUrlToDBAttributes (
632 const fileUrls = videoObject.url.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] 653 videoOrPlaylist: MVideo | MStreamingPlaylist,
654 urls: (ActivityTagObject | ActivityUrlObject)[]
655) {
656 const fileUrls = urls.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
633 657
634 if (fileUrls.length === 0) { 658 if (fileUrls.length === 0) return []
635 throw new Error('Cannot find video files for ' + video.url)
636 }
637 659
638 const attributes: FilteredModelAttributes<VideoFileModel>[] = [] 660 const attributes: FilteredModelAttributes<VideoFileModel>[] = []
639 for (const fileUrl of fileUrls) { 661 for (const fileUrl of fileUrls) {
640 // Fetch associated magnet uri 662 // Fetch associated magnet uri
641 const magnet = videoObject.url.find(u => { 663 const magnet = urls.filter(isAPMagnetUrlObject)
642 const mediaType = u.mediaType || u.mimeType 664 .find(u => u.height === fileUrl.height)
643 return mediaType === 'application/x-bittorrent;x-scheme-handler/magnet' && (u as any).height === fileUrl.height
644 })
645 665
646 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) 666 if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href)
647 667
@@ -650,14 +670,17 @@ function videoFileActivityUrlToDBAttributes (video: MVideo, videoObject: VideoTo
650 throw new Error('Cannot parse magnet URI ' + magnet.href) 670 throw new Error('Cannot parse magnet URI ' + magnet.href)
651 } 671 }
652 672
653 const mediaType = fileUrl.mediaType || fileUrl.mimeType 673 const mediaType = fileUrl.mediaType
654 const attribute = { 674 const attribute = {
655 extname: MIMETYPES.VIDEO.MIMETYPE_EXT[ mediaType ], 675 extname: MIMETYPES.VIDEO.MIMETYPE_EXT[ mediaType ],
656 infoHash: parsed.infoHash, 676 infoHash: parsed.infoHash,
657 resolution: fileUrl.height, 677 resolution: fileUrl.height,
658 size: fileUrl.size, 678 size: fileUrl.size,
659 videoId: video.id, 679 fps: fileUrl.fps || -1,
660 fps: fileUrl.fps || -1 680
681 // This is a video file owned by a video or by a streaming playlist
682 videoId: (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? null : videoOrPlaylist.id,
683 videoStreamingPlaylistId: (videoOrPlaylist as MStreamingPlaylist).playlistUrl ? videoOrPlaylist.id : null
661 } 684 }
662 685
663 attributes.push(attribute) 686 attributes.push(attribute)
@@ -670,12 +693,15 @@ function streamingPlaylistActivityUrlToDBAttributes (video: MVideoId, videoObjec
670 const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[] 693 const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
671 if (playlistUrls.length === 0) return [] 694 if (playlistUrls.length === 0) return []
672 695
673 const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = [] 696 const attributes: (FilteredModelAttributes<VideoStreamingPlaylistModel> & { tagAPObject?: ActivityTagObject[] })[] = []
674 for (const playlistUrlObject of playlistUrls) { 697 for (const playlistUrlObject of playlistUrls) {
675 const segmentsSha256UrlObject = playlistUrlObject.tag 698 const segmentsSha256UrlObject = playlistUrlObject.tag.find(isAPPlaylistSegmentHashesUrlObject)
676 .find(t => { 699
677 return isAPPlaylistSegmentHashesUrlObject(t) 700 let files: unknown[] = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[]
678 }) as ActivityPlaylistSegmentHashesObject 701
702 // FIXME: backward compatibility introduced in v2.1.0
703 if (files.length === 0) files = videoFiles
704
679 if (!segmentsSha256UrlObject) { 705 if (!segmentsSha256UrlObject) {
680 logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject }) 706 logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject })
681 continue 707 continue
@@ -685,9 +711,10 @@ function streamingPlaylistActivityUrlToDBAttributes (video: MVideoId, videoObjec
685 type: VideoStreamingPlaylistType.HLS, 711 type: VideoStreamingPlaylistType.HLS,
686 playlistUrl: playlistUrlObject.href, 712 playlistUrl: playlistUrlObject.href,
687 segmentsSha256Url: segmentsSha256UrlObject.href, 713 segmentsSha256Url: segmentsSha256UrlObject.href,
688 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, videoFiles), 714 p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files),
689 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, 715 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
690 videoId: video.id 716 videoId: video.id,
717 tagAPObject: playlistUrlObject.tag
691 } 718 }
692 719
693 attributes.push(attribute) 720 attributes.push(attribute)
diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index 05136c21c..943721dd7 100644
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -12,6 +12,7 @@ import { VideoFileModel } from '../models/video/video-file'
12import { CONFIG } from '../initializers/config' 12import { CONFIG } from '../initializers/config'
13import { sequelizeTypescript } from '../initializers/database' 13import { sequelizeTypescript } from '../initializers/database'
14import { MVideoWithFile } from '@server/typings/models' 14import { MVideoWithFile } from '@server/typings/models'
15import { getVideoFilename, getVideoFilePath } from './video-paths'
15 16
16async function updateStreamingPlaylistsInfohashesIfNeeded () { 17async function updateStreamingPlaylistsInfohashesIfNeeded () {
17 const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() 18 const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
@@ -32,13 +33,14 @@ async function updateMasterHLSPlaylist (video: MVideoWithFile) {
32 const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) 33 const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
33 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] 34 const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ]
34 const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename()) 35 const masterPlaylistPath = join(directory, VideoStreamingPlaylistModel.getMasterHlsPlaylistFilename())
36 const streamingPlaylist = video.getHLSPlaylist()
35 37
36 for (const file of video.VideoFiles) { 38 for (const file of streamingPlaylist.VideoFiles) {
37 // If we did not generated a playlist for this resolution, skip 39 // If we did not generated a playlist for this resolution, skip
38 const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) 40 const filePlaylistPath = join(directory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
39 if (await pathExists(filePlaylistPath) === false) continue 41 if (await pathExists(filePlaylistPath) === false) continue
40 42
41 const videoFilePath = video.getVideoFilePath(file) 43 const videoFilePath = getVideoFilePath(streamingPlaylist, file)
42 44
43 const size = await getVideoFileSize(videoFilePath) 45 const size = await getVideoFileSize(videoFilePath)
44 46
@@ -59,12 +61,13 @@ async function updateSha256Segments (video: MVideoWithFile) {
59 const json: { [filename: string]: { [range: string]: string } } = {} 61 const json: { [filename: string]: { [range: string]: string } } = {}
60 62
61 const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) 63 const playlistDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
64 const hlsPlaylist = video.getHLSPlaylist()
62 65
63 // For all the resolutions available for this video 66 // For all the resolutions available for this video
64 for (const file of video.VideoFiles) { 67 for (const file of hlsPlaylist.VideoFiles) {
65 const rangeHashes: { [range: string]: string } = {} 68 const rangeHashes: { [range: string]: string } = {}
66 69
67 const videoPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution)) 70 const videoPath = getVideoFilePath(hlsPlaylist, file)
68 const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution)) 71 const playlistPath = join(playlistDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(file.resolution))
69 72
70 // Maybe the playlist is not generated for this resolution yet 73 // Maybe the playlist is not generated for this resolution yet
@@ -82,7 +85,7 @@ async function updateSha256Segments (video: MVideoWithFile) {
82 } 85 }
83 await close(fd) 86 await close(fd)
84 87
85 const videoFilename = VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, file.resolution) 88 const videoFilename = getVideoFilename(hlsPlaylist, file)
86 json[videoFilename] = rangeHashes 89 json[videoFilename] = rangeHashes
87 } 90 }
88 91
diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts
index 5c5b7dccb..99c991e72 100644
--- a/server/lib/job-queue/handlers/video-file-import.ts
+++ b/server/lib/job-queue/handlers/video-file-import.ts
@@ -7,6 +7,8 @@ import { copy, stat } from 'fs-extra'
7import { VideoFileModel } from '../../../models/video/video-file' 7import { VideoFileModel } from '../../../models/video/video-file'
8import { extname } from 'path' 8import { extname } from 'path'
9import { MVideoFile, MVideoWithFile } from '@server/typings/models' 9import { MVideoFile, MVideoWithFile } from '@server/typings/models'
10import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
11import { getVideoFilePath } from '@server/lib/video-paths'
10 12
11export type VideoFileImportPayload = { 13export type VideoFileImportPayload = {
12 videoUUID: string, 14 videoUUID: string,
@@ -68,10 +70,10 @@ async function updateVideoFile (video: MVideoWithFile, inputFilePath: string) {
68 updatedVideoFile = currentVideoFile 70 updatedVideoFile = currentVideoFile
69 } 71 }
70 72
71 const outputPath = video.getVideoFilePath(updatedVideoFile) 73 const outputPath = getVideoFilePath(video, updatedVideoFile)
72 await copy(inputFilePath, outputPath) 74 await copy(inputFilePath, outputPath)
73 75
74 await video.createTorrentAndSetInfoHash(updatedVideoFile) 76 await createTorrentAndSetInfoHash(video, updatedVideoFile)
75 77
76 await updatedVideoFile.save() 78 await updatedVideoFile.save()
77 79
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index 93a3e9d90..1fca17584 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -4,14 +4,14 @@ import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl'
4import { VideoImportModel } from '../../../models/video/video-import' 4import { VideoImportModel } from '../../../models/video/video-import'
5import { VideoImportState } from '../../../../shared/models/videos' 5import { VideoImportState } from '../../../../shared/models/videos'
6import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' 6import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
7import { extname, join } from 'path' 7import { extname } from 'path'
8import { VideoFileModel } from '../../../models/video/video-file' 8import { VideoFileModel } from '../../../models/video/video-file'
9import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants' 9import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
10import { VideoState } from '../../../../shared' 10import { VideoState } from '../../../../shared'
11import { JobQueue } from '../index' 11import { JobQueue } from '../index'
12import { federateVideoIfNeeded } from '../../activitypub' 12import { federateVideoIfNeeded } from '../../activitypub'
13import { VideoModel } from '../../../models/video/video' 13import { VideoModel } from '../../../models/video/video'
14import { downloadWebTorrentVideo } from '../../../helpers/webtorrent' 14import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
15import { getSecureTorrentName } from '../../../helpers/utils' 15import { getSecureTorrentName } from '../../../helpers/utils'
16import { move, remove, stat } from 'fs-extra' 16import { move, remove, stat } from 'fs-extra'
17import { Notifier } from '../../notifier' 17import { Notifier } from '../../notifier'
@@ -21,7 +21,7 @@ import { createVideoMiniatureFromUrl, generateVideoMiniature } from '../../thumb
21import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type' 21import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
22import { MThumbnail } from '../../../typings/models/video/thumbnail' 22import { MThumbnail } from '../../../typings/models/video/thumbnail'
23import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import' 23import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import'
24import { MVideoBlacklistVideo, MVideoBlacklist } from '@server/typings/models' 24import { getVideoFilePath } from '@server/lib/video-paths'
25 25
26type VideoImportYoutubeDLPayload = { 26type VideoImportYoutubeDLPayload = {
27 type: 'youtube-dl' 27 type: 'youtube-dl'
@@ -142,12 +142,12 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
142 } 142 }
143 videoFile = new VideoFileModel(videoFileData) 143 videoFile = new VideoFileModel(videoFileData)
144 144
145 const videoWithFiles = Object.assign(videoImport.Video, { VideoFiles: [ videoFile ] }) 145 const videoWithFiles = Object.assign(videoImport.Video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] })
146 // To clean files if the import fails 146 // To clean files if the import fails
147 const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles }) 147 const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles })
148 148
149 // Move file 149 // Move file
150 videoDestFile = join(CONFIG.STORAGE.VIDEOS_DIR, videoImportWithFiles.Video.getVideoFilename(videoFile)) 150 videoDestFile = getVideoFilePath(videoImportWithFiles.Video, videoFile)
151 await move(tempVideoPath, videoDestFile) 151 await move(tempVideoPath, videoDestFile)
152 tempVideoPath = null // This path is not used anymore 152 tempVideoPath = null // This path is not used anymore
153 153
@@ -168,7 +168,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
168 } 168 }
169 169
170 // Create torrent 170 // Create torrent
171 await videoImportWithFiles.Video.createTorrentAndSetInfoHash(videoFile) 171 await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile)
172 172
173 const { videoImportUpdated, video } = await sequelizeTypescript.transaction(async t => { 173 const { videoImportUpdated, video } = await sequelizeTypescript.transaction(async t => {
174 const videoImportToUpdate = videoImportWithFiles as MVideoImportVideo 174 const videoImportToUpdate = videoImportWithFiles as MVideoImportVideo
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts
index 2ebe15bcb..39b9fac98 100644
--- a/server/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/lib/job-queue/handlers/video-transcoding.ts
@@ -1,5 +1,5 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { VideoResolution, VideoState } from '../../../../shared' 2import { VideoResolution } from '../../../../shared'
3import { logger } from '../../../helpers/logger' 3import { logger } from '../../../helpers/logger'
4import { VideoModel } from '../../../models/video/video' 4import { VideoModel } from '../../../models/video/video'
5import { JobQueue } from '../job-queue' 5import { JobQueue } from '../job-queue'
@@ -8,10 +8,10 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils'
8import { sequelizeTypescript } from '../../../initializers' 8import { sequelizeTypescript } from '../../../initializers'
9import * as Bluebird from 'bluebird' 9import * as Bluebird from 'bluebird'
10import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils' 10import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils'
11import { generateHlsPlaylist, optimizeVideofile, transcodeOriginalVideofile, mergeAudioVideofile } from '../../video-transcoding' 11import { generateHlsPlaylist, mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewResolution } from '../../video-transcoding'
12import { Notifier } from '../../notifier' 12import { Notifier } from '../../notifier'
13import { CONFIG } from '../../../initializers/config' 13import { CONFIG } from '../../../initializers/config'
14import { MVideoUUID, MVideoWithFile } from '@server/typings/models' 14import { MVideoFullLight, MVideoUUID, MVideoWithFile } from '@server/typings/models'
15 15
16interface BaseTranscodingPayload { 16interface BaseTranscodingPayload {
17 videoUUID: string 17 videoUUID: string
@@ -22,6 +22,7 @@ interface HLSTranscodingPayload extends BaseTranscodingPayload {
22 type: 'hls' 22 type: 'hls'
23 isPortraitMode?: boolean 23 isPortraitMode?: boolean
24 resolution: VideoResolution 24 resolution: VideoResolution
25 copyCodecs: boolean
25} 26}
26 27
27interface NewResolutionTranscodingPayload extends BaseTranscodingPayload { 28interface NewResolutionTranscodingPayload extends BaseTranscodingPayload {
@@ -54,11 +55,11 @@ async function processVideoTranscoding (job: Bull.Job) {
54 } 55 }
55 56
56 if (payload.type === 'hls') { 57 if (payload.type === 'hls') {
57 await generateHlsPlaylist(video, payload.resolution, payload.isPortraitMode || false) 58 await generateHlsPlaylist(video, payload.resolution, payload.copyCodecs, payload.isPortraitMode || false)
58 59
59 await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video) 60 await retryTransactionWrapper(onHlsPlaylistGenerationSuccess, video)
60 } else if (payload.type === 'new-resolution') { 61 } else if (payload.type === 'new-resolution') {
61 await transcodeOriginalVideofile(video, payload.resolution, payload.isPortraitMode || false) 62 await transcodeNewResolution(video, payload.resolution, payload.isPortraitMode || false)
62 63
63 await retryTransactionWrapper(publishNewResolutionIfNeeded, video, payload) 64 await retryTransactionWrapper(publishNewResolutionIfNeeded, video, payload)
64 } else if (payload.type === 'merge-audio') { 65 } else if (payload.type === 'merge-audio') {
@@ -66,7 +67,7 @@ async function processVideoTranscoding (job: Bull.Job) {
66 67
67 await retryTransactionWrapper(publishNewResolutionIfNeeded, video, payload) 68 await retryTransactionWrapper(publishNewResolutionIfNeeded, video, payload)
68 } else { 69 } else {
69 await optimizeVideofile(video) 70 await optimizeOriginalVideofile(video)
70 71
71 await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload) 72 await retryTransactionWrapper(onVideoFileOptimizerSuccess, video, payload)
72 } 73 }
@@ -74,48 +75,24 @@ async function processVideoTranscoding (job: Bull.Job) {
74 return video 75 return video
75} 76}
76 77
77async function onHlsPlaylistGenerationSuccess (video: MVideoUUID) { 78async function onHlsPlaylistGenerationSuccess (video: MVideoFullLight) {
78 if (video === undefined) return undefined 79 if (video === undefined) return undefined
79 80
80 await sequelizeTypescript.transaction(async t => { 81 // We generated the HLS playlist, we don't need the webtorrent files anymore if the admin disabled it
81 // Maybe the video changed in database, refresh it 82 if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false) {
82 let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t) 83 for (const file of video.VideoFiles) {
83 // Video does not exist anymore 84 await video.removeFile(file)
84 if (!videoDatabase) return undefined 85 await file.destroy()
85
86 // If the video was not published, we consider it is a new one for other instances
87 await federateVideoIfNeeded(videoDatabase, false, t)
88 })
89}
90
91async function publishNewResolutionIfNeeded (video: MVideoUUID, payload?: NewResolutionTranscodingPayload | MergeAudioTranscodingPayload) {
92 const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => {
93 // Maybe the video changed in database, refresh it
94 let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
95 // Video does not exist anymore
96 if (!videoDatabase) return undefined
97
98 let videoPublished = false
99
100 // We transcoded the video file in another format, now we can publish it
101 if (videoDatabase.state !== VideoState.PUBLISHED) {
102 videoPublished = true
103
104 videoDatabase.state = VideoState.PUBLISHED
105 videoDatabase.publishedAt = new Date()
106 videoDatabase = await videoDatabase.save({ transaction: t })
107 } 86 }
108 87
109 // If the video was not published, we consider it is a new one for other instances 88 video.VideoFiles = []
110 await federateVideoIfNeeded(videoDatabase, videoPublished, t) 89 }
111 90
112 return { videoDatabase, videoPublished } 91 return publishAndFederateIfNeeded(video)
113 }) 92}
114 93
115 if (videoPublished) { 94async function publishNewResolutionIfNeeded (video: MVideoUUID, payload?: NewResolutionTranscodingPayload | MergeAudioTranscodingPayload) {
116 Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase) 95 await publishAndFederateIfNeeded(video)
117 Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase)
118 }
119 96
120 await createHlsJobIfEnabled(payload) 97 await createHlsJobIfEnabled(payload)
121} 98}
@@ -124,7 +101,7 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O
124 if (videoArg === undefined) return undefined 101 if (videoArg === undefined) return undefined
125 102
126 // Outside the transaction (IO on disk) 103 // Outside the transaction (IO on disk)
127 const { videoFileResolution } = await videoArg.getOriginalFileResolution() 104 const { videoFileResolution } = await videoArg.getMaxQualityResolution()
128 105
129 const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => { 106 const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => {
130 // Maybe the video changed in database, refresh it 107 // Maybe the video changed in database, refresh it
@@ -141,14 +118,29 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O
141 118
142 let videoPublished = false 119 let videoPublished = false
143 120
121 const hlsPayload = Object.assign({}, payload, { resolution: videoDatabase.getMaxQualityFile().resolution })
122 await createHlsJobIfEnabled(hlsPayload)
123
144 if (resolutionsEnabled.length !== 0) { 124 if (resolutionsEnabled.length !== 0) {
145 const tasks: (Bluebird<Bull.Job<any>> | Promise<Bull.Job<any>>)[] = [] 125 const tasks: (Bluebird<Bull.Job<any>> | Promise<Bull.Job<any>>)[] = []
146 126
147 for (const resolution of resolutionsEnabled) { 127 for (const resolution of resolutionsEnabled) {
148 const dataInput = { 128 let dataInput: VideoTranscodingPayload
149 type: 'new-resolution' as 'new-resolution', 129
150 videoUUID: videoDatabase.uuid, 130 if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED) {
151 resolution 131 dataInput = {
132 type: 'new-resolution' as 'new-resolution',
133 videoUUID: videoDatabase.uuid,
134 resolution
135 }
136 } else if (CONFIG.TRANSCODING.HLS.ENABLED) {
137 dataInput = {
138 type: 'hls',
139 videoUUID: videoDatabase.uuid,
140 resolution,
141 isPortraitMode: false,
142 copyCodecs: false
143 }
152 } 144 }
153 145
154 const p = JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput }) 146 const p = JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput })
@@ -159,11 +151,8 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O
159 151
160 logger.info('Transcoding jobs created for uuid %s.', videoDatabase.uuid, { resolutionsEnabled }) 152 logger.info('Transcoding jobs created for uuid %s.', videoDatabase.uuid, { resolutionsEnabled })
161 } else { 153 } else {
162 videoPublished = true
163
164 // No transcoding to do, it's now published 154 // No transcoding to do, it's now published
165 videoDatabase.state = VideoState.PUBLISHED 155 videoPublished = await videoDatabase.publishIfNeededAndSave(t)
166 videoDatabase = await videoDatabase.save({ transaction: t })
167 156
168 logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy }) 157 logger.info('No transcoding jobs created for video %s (no resolutions).', videoDatabase.uuid, { privacy: videoDatabase.privacy })
169 } 158 }
@@ -175,9 +164,6 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O
175 164
176 if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase) 165 if (payload.isNewVideo) Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase)
177 if (videoPublished) Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase) 166 if (videoPublished) Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase)
178
179 const hlsPayload = Object.assign({}, payload, { resolution: videoDatabase.getOriginalFile().resolution })
180 await createHlsJobIfEnabled(hlsPayload)
181} 167}
182 168
183// --------------------------------------------------------------------------- 169// ---------------------------------------------------------------------------
@@ -196,9 +182,32 @@ function createHlsJobIfEnabled (payload?: { videoUUID: string, resolution: numbe
196 type: 'hls' as 'hls', 182 type: 'hls' as 'hls',
197 videoUUID: payload.videoUUID, 183 videoUUID: payload.videoUUID,
198 resolution: payload.resolution, 184 resolution: payload.resolution,
199 isPortraitMode: payload.isPortraitMode 185 isPortraitMode: payload.isPortraitMode,
186 copyCodecs: true
200 } 187 }
201 188
202 return JobQueue.Instance.createJob({ type: 'video-transcoding', payload: hlsTranscodingPayload }) 189 return JobQueue.Instance.createJob({ type: 'video-transcoding', payload: hlsTranscodingPayload })
203 } 190 }
204} 191}
192
193async function publishAndFederateIfNeeded (video: MVideoUUID) {
194 const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => {
195 // Maybe the video changed in database, refresh it
196 const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(video.uuid, t)
197 // Video does not exist anymore
198 if (!videoDatabase) return undefined
199
200 // We transcoded the video file in another format, now we can publish it
201 const videoPublished = await videoDatabase.publishIfNeededAndSave(t)
202
203 // If the video was not published, we consider it is a new one for other instances
204 await federateVideoIfNeeded(videoDatabase, videoPublished, t)
205
206 return { videoDatabase, videoPublished }
207 })
208
209 if (videoPublished) {
210 Notifier.Instance.notifyOnNewVideoIfNeeded(videoDatabase)
211 Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(videoDatabase)
212 }
213}
diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts
index 5b673b913..293bba91f 100644
--- a/server/lib/schedulers/update-videos-scheduler.ts
+++ b/server/lib/schedulers/update-videos-scheduler.ts
@@ -6,8 +6,8 @@ import { federateVideoIfNeeded } from '../activitypub'
6import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' 6import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
7import { VideoPrivacy } from '../../../shared/models/videos' 7import { VideoPrivacy } from '../../../shared/models/videos'
8import { Notifier } from '../notifier' 8import { Notifier } from '../notifier'
9import { VideoModel } from '../../models/video/video'
10import { sequelizeTypescript } from '../../initializers/database' 9import { sequelizeTypescript } from '../../initializers/database'
10import { MVideoFullLight } from '@server/typings/models'
11 11
12export class UpdateVideosScheduler extends AbstractScheduler { 12export class UpdateVideosScheduler extends AbstractScheduler {
13 13
@@ -28,7 +28,7 @@ export class UpdateVideosScheduler extends AbstractScheduler {
28 28
29 const publishedVideos = await sequelizeTypescript.transaction(async t => { 29 const publishedVideos = await sequelizeTypescript.transaction(async t => {
30 const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate(t) 30 const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate(t)
31 const publishedVideos: VideoModel[] = [] 31 const publishedVideos: MVideoFullLight[] = []
32 32
33 for (const schedule of schedules) { 33 for (const schedule of schedules) {
34 const video = schedule.Video 34 const video = schedule.Video
@@ -45,8 +45,8 @@ export class UpdateVideosScheduler extends AbstractScheduler {
45 await federateVideoIfNeeded(video, isNewVideo, t) 45 await federateVideoIfNeeded(video, isNewVideo, t)
46 46
47 if (oldPrivacy === VideoPrivacy.UNLISTED || oldPrivacy === VideoPrivacy.PRIVATE) { 47 if (oldPrivacy === VideoPrivacy.UNLISTED || oldPrivacy === VideoPrivacy.PRIVATE) {
48 video.ScheduleVideoUpdate = schedule 48 const videoToPublish: MVideoFullLight = Object.assign(video, { ScheduleVideoUpdate: schedule, UserVideoHistories: [] })
49 publishedVideos.push(video) 49 publishedVideos.push(videoToPublish)
50 } 50 }
51 } 51 }
52 52
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index 1e30f6ebc..f2bd75cb4 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -3,7 +3,7 @@ import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER }
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { VideosRedundancy } from '../../../shared/models/redundancy' 4import { VideosRedundancy } from '../../../shared/models/redundancy'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
6import { downloadWebTorrentVideo } from '../../helpers/webtorrent' 6import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent'
7import { join } from 'path' 7import { join } from 'path'
8import { move } from 'fs-extra' 8import { move } from 'fs-extra'
9import { getServerActor } from '../../helpers/utils' 9import { getServerActor } from '../../helpers/utils'
@@ -24,6 +24,7 @@ import {
24 MVideoRedundancyVideo, 24 MVideoRedundancyVideo,
25 MVideoWithAllFiles 25 MVideoWithAllFiles
26} from '@server/typings/models' 26} from '@server/typings/models'
27import { getVideoFilename } from '../video-paths'
27 28
28type CandidateToDuplicate = { 29type CandidateToDuplicate = {
29 redundancy: VideosRedundancy, 30 redundancy: VideosRedundancy,
@@ -195,11 +196,11 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
195 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) 196 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy)
196 197
197 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() 198 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
198 const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) 199 const magnetUri = await generateMagnetUri(video, file, baseUrlHttp, baseUrlWs)
199 200
200 const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) 201 const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT)
201 202
202 const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file)) 203 const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, getVideoFilename(video, file))
203 await move(tmpPath, destPath, { overwrite: true }) 204 await move(tmpPath, destPath, { overwrite: true })
204 205
205 const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ 206 const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts
index 84791955e..a99f71629 100644
--- a/server/lib/thumbnail.ts
+++ b/server/lib/thumbnail.ts
@@ -9,6 +9,7 @@ import { downloadImage } from '../helpers/requests'
9import { MVideoPlaylistThumbnail } from '../typings/models/video/video-playlist' 9import { MVideoPlaylistThumbnail } from '../typings/models/video/video-playlist'
10import { MVideoFile, MVideoThumbnail } from '../typings/models' 10import { MVideoFile, MVideoThumbnail } from '../typings/models'
11import { MThumbnail } from '../typings/models/video/thumbnail' 11import { MThumbnail } from '../typings/models/video/thumbnail'
12import { getVideoFilePath } from './video-paths'
12 13
13type ImageSize = { height: number, width: number } 14type ImageSize = { height: number, width: number }
14 15
@@ -55,7 +56,7 @@ function createVideoMiniatureFromExisting (
55} 56}
56 57
57function generateVideoMiniature (video: MVideoThumbnail, videoFile: MVideoFile, type: ThumbnailType) { 58function generateVideoMiniature (video: MVideoThumbnail, videoFile: MVideoFile, type: ThumbnailType) {
58 const input = video.getVideoFilePath(videoFile) 59 const input = getVideoFilePath(video, videoFile)
59 60
60 const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) 61 const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type)
61 const thumbnailCreator = videoFile.isAudio() 62 const thumbnailCreator = videoFile.isAudio()
diff --git a/server/lib/video-paths.ts b/server/lib/video-paths.ts
new file mode 100644
index 000000000..63011cdb2
--- /dev/null
+++ b/server/lib/video-paths.ts
@@ -0,0 +1,64 @@
1import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/typings/models'
2import { extractVideo } from './videos'
3import { join } from 'path'
4import { CONFIG } from '@server/initializers/config'
5import { HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants'
6
7// ################## Video file name ##################
8
9function getVideoFilename (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
10 const video = extractVideo(videoOrPlaylist)
11
12 if (isStreamingPlaylist(videoOrPlaylist)) {
13 return generateVideoStreamingPlaylistName(video.uuid, videoFile.resolution)
14 }
15
16 return generateWebTorrentVideoName(video.uuid, videoFile.resolution, videoFile.extname)
17}
18
19function generateVideoStreamingPlaylistName (uuid: string, resolution: number) {
20 return `${uuid}-${resolution}-fragmented.mp4`
21}
22
23function generateWebTorrentVideoName (uuid: string, resolution: number, extname: string) {
24 return uuid + '-' + resolution + extname
25}
26
27function getVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile, isRedundancy = false) {
28 if (isStreamingPlaylist(videoOrPlaylist)) {
29 const video = extractVideo(videoOrPlaylist)
30 return join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid, getVideoFilename(videoOrPlaylist, videoFile))
31 }
32
33 const baseDir = isRedundancy ? CONFIG.STORAGE.REDUNDANCY_DIR : CONFIG.STORAGE.VIDEOS_DIR
34 return join(baseDir, getVideoFilename(videoOrPlaylist, videoFile))
35}
36
37// ################## Torrents ##################
38
39function getTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
40 const video = extractVideo(videoOrPlaylist)
41 const extension = '.torrent'
42
43 if (isStreamingPlaylist(videoOrPlaylist)) {
44 return `${video.uuid}-${videoFile.resolution}-${videoOrPlaylist.getStringType()}${extension}`
45 }
46
47 return video.uuid + '-' + videoFile.resolution + extension
48}
49
50function getTorrentFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
51 return join(CONFIG.STORAGE.TORRENTS_DIR, getTorrentFileName(videoOrPlaylist, videoFile))
52}
53
54// ---------------------------------------------------------------------------
55
56export {
57 generateVideoStreamingPlaylistName,
58 generateWebTorrentVideoName,
59 getVideoFilename,
60 getVideoFilePath,
61
62 getTorrentFileName,
63 getTorrentFilePath
64}
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts
index 612d388ee..9243d1742 100644
--- a/server/lib/video-transcoding.ts
+++ b/server/lib/video-transcoding.ts
@@ -1,5 +1,5 @@
1import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, WEBSERVER } from '../initializers/constants' 1import { HLS_STREAMING_PLAYLIST_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, WEBSERVER } from '../initializers/constants'
2import { basename, join } from 'path' 2import { basename, extname as extnameUtil, join } from 'path'
3import { 3import {
4 canDoQuickTranscode, 4 canDoQuickTranscode,
5 getDurationFromVideoFile, 5 getDurationFromVideoFile,
@@ -16,18 +16,19 @@ import { updateMasterHLSPlaylist, updateSha256Segments } from './hls'
16import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' 16import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist'
17import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type' 17import { VideoStreamingPlaylistType } from '../../shared/models/videos/video-streaming-playlist.type'
18import { CONFIG } from '../initializers/config' 18import { CONFIG } from '../initializers/config'
19import { MVideoFile, MVideoWithFile, MVideoWithFileThumbnail } from '@server/typings/models' 19import { MStreamingPlaylistFilesVideo, MVideoFile, MVideoWithAllFiles, MVideoWithFile } from '@server/typings/models'
20import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
21import { generateVideoStreamingPlaylistName, getVideoFilename, getVideoFilePath } from './video-paths'
20 22
21/** 23/**
22 * Optimize the original video file and replace it. The resolution is not changed. 24 * Optimize the original video file and replace it. The resolution is not changed.
23 */ 25 */
24async function optimizeVideofile (video: MVideoWithFile, inputVideoFileArg?: MVideoFile) { 26async function optimizeOriginalVideofile (video: MVideoWithFile, inputVideoFileArg?: MVideoFile) {
25 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
26 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 27 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
27 const newExtname = '.mp4' 28 const newExtname = '.mp4'
28 29
29 const inputVideoFile = inputVideoFileArg ? inputVideoFileArg : video.getOriginalFile() 30 const inputVideoFile = inputVideoFileArg || video.getMaxQualityFile()
30 const videoInputPath = join(videosDirectory, video.getVideoFilename(inputVideoFile)) 31 const videoInputPath = getVideoFilePath(video, inputVideoFile)
31 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) 32 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
32 33
33 const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath) 34 const transcodeType: TranscodeOptionsType = await canDoQuickTranscode(videoInputPath)
@@ -35,7 +36,7 @@ async function optimizeVideofile (video: MVideoWithFile, inputVideoFileArg?: MVi
35 : 'video' 36 : 'video'
36 37
37 const transcodeOptions: TranscodeOptions = { 38 const transcodeOptions: TranscodeOptions = {
38 type: transcodeType as any, // FIXME: typing issue 39 type: transcodeType,
39 inputPath: videoInputPath, 40 inputPath: videoInputPath,
40 outputPath: videoTranscodedPath, 41 outputPath: videoTranscodedPath,
41 resolution: inputVideoFile.resolution 42 resolution: inputVideoFile.resolution
@@ -50,7 +51,7 @@ async function optimizeVideofile (video: MVideoWithFile, inputVideoFileArg?: MVi
50 // Important to do this before getVideoFilename() to take in account the new file extension 51 // Important to do this before getVideoFilename() to take in account the new file extension
51 inputVideoFile.extname = newExtname 52 inputVideoFile.extname = newExtname
52 53
53 const videoOutputPath = video.getVideoFilePath(inputVideoFile) 54 const videoOutputPath = getVideoFilePath(video, inputVideoFile)
54 55
55 await onVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) 56 await onVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath)
56 } catch (err) { 57 } catch (err) {
@@ -64,13 +65,12 @@ async function optimizeVideofile (video: MVideoWithFile, inputVideoFileArg?: MVi
64/** 65/**
65 * Transcode the original video file to a lower resolution. 66 * Transcode the original video file to a lower resolution.
66 */ 67 */
67async function transcodeOriginalVideofile (video: MVideoWithFile, resolution: VideoResolution, isPortrait: boolean) { 68async function transcodeNewResolution (video: MVideoWithFile, resolution: VideoResolution, isPortrait: boolean) {
68 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
69 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 69 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
70 const extname = '.mp4' 70 const extname = '.mp4'
71 71
72 // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed 72 // We are sure it's x264 in mp4 because optimizeOriginalVideofile was already executed
73 const videoInputPath = join(videosDirectory, video.getVideoFilename(video.getOriginalFile())) 73 const videoInputPath = getVideoFilePath(video, video.getMaxQualityFile())
74 74
75 const newVideoFile = new VideoFileModel({ 75 const newVideoFile = new VideoFileModel({
76 resolution, 76 resolution,
@@ -78,8 +78,8 @@ async function transcodeOriginalVideofile (video: MVideoWithFile, resolution: Vi
78 size: 0, 78 size: 0,
79 videoId: video.id 79 videoId: video.id
80 }) 80 })
81 const videoOutputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(newVideoFile)) 81 const videoOutputPath = getVideoFilePath(video, newVideoFile)
82 const videoTranscodedPath = join(transcodeDirectory, video.getVideoFilename(newVideoFile)) 82 const videoTranscodedPath = join(transcodeDirectory, getVideoFilename(video, newVideoFile))
83 83
84 const transcodeOptions = { 84 const transcodeOptions = {
85 type: 'video' as 'video', 85 type: 'video' as 'video',
@@ -94,14 +94,13 @@ async function transcodeOriginalVideofile (video: MVideoWithFile, resolution: Vi
94 return onVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath) 94 return onVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath)
95} 95}
96 96
97async function mergeAudioVideofile (video: MVideoWithFileThumbnail, resolution: VideoResolution) { 97async function mergeAudioVideofile (video: MVideoWithAllFiles, resolution: VideoResolution) {
98 const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR
99 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR 98 const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
100 const newExtname = '.mp4' 99 const newExtname = '.mp4'
101 100
102 const inputVideoFile = video.getOriginalFile() 101 const inputVideoFile = video.getMaxQualityFile()
103 102
104 const audioInputPath = join(videosDirectory, video.getVideoFilename(video.getOriginalFile())) 103 const audioInputPath = getVideoFilePath(video, inputVideoFile)
105 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) 104 const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
106 105
107 // If the user updates the video preview during transcoding 106 // If the user updates the video preview during transcoding
@@ -130,7 +129,7 @@ async function mergeAudioVideofile (video: MVideoWithFileThumbnail, resolution:
130 // Important to do this before getVideoFilename() to take in account the new file extension 129 // Important to do this before getVideoFilename() to take in account the new file extension
131 inputVideoFile.extname = newExtname 130 inputVideoFile.extname = newExtname
132 131
133 const videoOutputPath = video.getVideoFilePath(inputVideoFile) 132 const videoOutputPath = getVideoFilePath(video, inputVideoFile)
134 // ffmpeg generated a new video file, so update the video duration 133 // ffmpeg generated a new video file, so update the video duration
135 // See https://trac.ffmpeg.org/ticket/5456 134 // See https://trac.ffmpeg.org/ticket/5456
136 video.duration = await getDurationFromVideoFile(videoTranscodedPath) 135 video.duration = await getDurationFromVideoFile(videoTranscodedPath)
@@ -139,33 +138,40 @@ async function mergeAudioVideofile (video: MVideoWithFileThumbnail, resolution:
139 return onVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath) 138 return onVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath)
140} 139}
141 140
142async function generateHlsPlaylist (video: MVideoWithFile, resolution: VideoResolution, isPortraitMode: boolean) { 141async function generateHlsPlaylist (video: MVideoWithFile, resolution: VideoResolution, copyCodecs: boolean, isPortraitMode: boolean) {
143 const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid) 142 const baseHlsDirectory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
144 await ensureDir(join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)) 143 await ensureDir(join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid))
145 144
146 const videoInputPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(video.getFile(resolution))) 145 const videoFileInput = copyCodecs
146 ? video.getWebTorrentFile(resolution)
147 : video.getMaxQualityFile()
148
149 const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist()
150 const videoInputPath = getVideoFilePath(videoOrStreamingPlaylist, videoFileInput)
151
147 const outputPath = join(baseHlsDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution)) 152 const outputPath = join(baseHlsDirectory, VideoStreamingPlaylistModel.getHlsPlaylistFilename(resolution))
153 const videoFilename = generateVideoStreamingPlaylistName(video.uuid, resolution)
148 154
149 const transcodeOptions = { 155 const transcodeOptions = {
150 type: 'hls' as 'hls', 156 type: 'hls' as 'hls',
151 inputPath: videoInputPath, 157 inputPath: videoInputPath,
152 outputPath, 158 outputPath,
153 resolution, 159 resolution,
160 copyCodecs,
154 isPortraitMode, 161 isPortraitMode,
155 162
156 hlsPlaylist: { 163 hlsPlaylist: {
157 videoFilename: VideoStreamingPlaylistModel.getHlsVideoName(video.uuid, resolution) 164 videoFilename
158 } 165 }
159 } 166 }
160 167
161 await transcode(transcodeOptions) 168 logger.debug('Will run transcode.', { transcodeOptions })
162 169
163 await updateMasterHLSPlaylist(video) 170 await transcode(transcodeOptions)
164 await updateSha256Segments(video)
165 171
166 const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid) 172 const playlistUrl = WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
167 173
168 await VideoStreamingPlaylistModel.upsert({ 174 const [ videoStreamingPlaylist ] = await VideoStreamingPlaylistModel.upsert({
169 videoId: video.id, 175 videoId: video.id,
170 playlistUrl, 176 playlistUrl,
171 segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid), 177 segmentsSha256Url: WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid),
@@ -173,15 +179,44 @@ async function generateHlsPlaylist (video: MVideoWithFile, resolution: VideoReso
173 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, 179 p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
174 180
175 type: VideoStreamingPlaylistType.HLS 181 type: VideoStreamingPlaylistType.HLS
182 }, { returning: true }) as [ MStreamingPlaylistFilesVideo, boolean ]
183 videoStreamingPlaylist.Video = video
184
185 const newVideoFile = new VideoFileModel({
186 resolution,
187 extname: extnameUtil(videoFilename),
188 size: 0,
189 fps: -1,
190 videoStreamingPlaylistId: videoStreamingPlaylist.id
176 }) 191 })
192
193 const videoFilePath = getVideoFilePath(videoStreamingPlaylist, newVideoFile)
194 const stats = await stat(videoFilePath)
195
196 newVideoFile.size = stats.size
197 newVideoFile.fps = await getVideoFileFPS(videoFilePath)
198
199 await createTorrentAndSetInfoHash(videoStreamingPlaylist, newVideoFile)
200
201 const updatedVideoFile = await newVideoFile.save()
202
203 videoStreamingPlaylist.VideoFiles = await videoStreamingPlaylist.$get('VideoFiles') as VideoFileModel[]
204 videoStreamingPlaylist.VideoFiles.push(updatedVideoFile)
205
206 video.setHLSPlaylist(videoStreamingPlaylist)
207
208 await updateMasterHLSPlaylist(video)
209 await updateSha256Segments(video)
210
211 return video
177} 212}
178 213
179// --------------------------------------------------------------------------- 214// ---------------------------------------------------------------------------
180 215
181export { 216export {
182 generateHlsPlaylist, 217 generateHlsPlaylist,
183 optimizeVideofile, 218 optimizeOriginalVideofile,
184 transcodeOriginalVideofile, 219 transcodeNewResolution,
185 mergeAudioVideofile 220 mergeAudioVideofile
186} 221}
187 222
@@ -196,7 +231,7 @@ async function onVideoFileTranscoding (video: MVideoWithFile, videoFile: MVideoF
196 videoFile.size = stats.size 231 videoFile.size = stats.size
197 videoFile.fps = fps 232 videoFile.fps = fps
198 233
199 await video.createTorrentAndSetInfoHash(videoFile) 234 await createTorrentAndSetInfoHash(video, videoFile)
200 235
201 const updatedVideoFile = await videoFile.save() 236 const updatedVideoFile = await videoFile.save()
202 237
diff --git a/server/lib/videos.ts b/server/lib/videos.ts
new file mode 100644
index 000000000..22e9afbf9
--- /dev/null
+++ b/server/lib/videos.ts
@@ -0,0 +1,11 @@
1import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/typings/models'
2
3function extractVideo (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
4 return isStreamingPlaylist(videoOrPlaylist)
5 ? videoOrPlaylist.Video
6 : videoOrPlaylist
7}
8
9export {
10 extractVideo
11}