aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2020-03-10 14:49:02 +0100
committerChocobozzz <me@florianbigard.com>2020-03-10 16:18:29 +0100
commit7b81edc854902a536083298472bf92bb6726edcf (patch)
tree822dae6b6d4755ac9afffe2569b74dcb45da5535 /server
parent8319d6ae72d4da6de51bd3d4b5c68040fc8dc3b4 (diff)
downloadPeerTube-7b81edc854902a536083298472bf92bb6726edcf.tar.gz
PeerTube-7b81edc854902a536083298472bf92bb6726edcf.tar.zst
PeerTube-7b81edc854902a536083298472bf92bb6726edcf.zip
Video file metadata PR cleanup
Diffstat (limited to 'server')
-rw-r--r--server/helpers/custom-validators/activitypub/videos.ts14
-rw-r--r--server/helpers/ffmpeg-utils.ts2
-rw-r--r--server/lib/activitypub/videos.ts26
-rw-r--r--server/lib/video-transcoding.ts3
-rw-r--r--server/models/video/video-file.ts79
-rw-r--r--server/models/video/video-format-utils.ts4
-rw-r--r--server/models/video/video.ts3
-rw-r--r--server/tests/api/videos/video-transcoder.ts70
8 files changed, 102 insertions, 99 deletions
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index af8c8a0c8..876cc7f50 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -13,6 +13,7 @@ import {
13import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc' 13import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc'
14import { VideoState } from '../../../../shared/models/videos' 14import { VideoState } from '../../../../shared/models/videos'
15import { logger } from '@server/helpers/logger' 15import { logger } from '@server/helpers/logger'
16import { ActivityVideoFileMetadataObject } from '@shared/models'
16 17
17function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { 18function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) {
18 return isBaseActivityValid(activity, 'Update') && 19 return isBaseActivityValid(activity, 'Update') &&
@@ -104,7 +105,15 @@ function isRemoteVideoUrlValid (url: any) {
104 (url.mediaType || url.mimeType) === 'application/x-mpegURL' && 105 (url.mediaType || url.mimeType) === 'application/x-mpegURL' &&
105 isActivityPubUrlValid(url.href) && 106 isActivityPubUrlValid(url.href) &&
106 isArray(url.tag) 107 isArray(url.tag)
107 ) 108 ) ||
109 isAPVideoFileMetadataObject(url)
110}
111
112function isAPVideoFileMetadataObject (url: any): url is ActivityVideoFileMetadataObject {
113 return url &&
114 url.type === 'Link' &&
115 url.mediaType === 'application/json' &&
116 isArray(url.rel) && url.rel.includes('metadata')
108} 117}
109 118
110// --------------------------------------------------------------------------- 119// ---------------------------------------------------------------------------
@@ -113,7 +122,8 @@ export {
113 sanitizeAndCheckVideoTorrentUpdateActivity, 122 sanitizeAndCheckVideoTorrentUpdateActivity,
114 isRemoteStringIdentifierValid, 123 isRemoteStringIdentifierValid,
115 sanitizeAndCheckVideoTorrentObject, 124 sanitizeAndCheckVideoTorrentObject,
116 isRemoteVideoUrlValid 125 isRemoteVideoUrlValid,
126 isAPVideoFileMetadataObject
117} 127}
118 128
119// --------------------------------------------------------------------------- 129// ---------------------------------------------------------------------------
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts
index 5ee295635..e0e408ea0 100644
--- a/server/helpers/ffmpeg-utils.ts
+++ b/server/helpers/ffmpeg-utils.ts
@@ -170,7 +170,7 @@ async function getVideoFileFPS (path: string) {
170 return 0 170 return 0
171} 171}
172 172
173async function getMetadataFromFile<T> (path: string, cb = metadata => metadata) { 173async function getMetadataFromFile <T> (path: string, cb = metadata => metadata) {
174 return new Promise<T>((res, rej) => { 174 return new Promise<T>((res, rej) => {
175 ffmpeg.ffprobe(path, (err, metadata) => { 175 ffmpeg.ffprobe(path, (err, metadata) => {
176 if (err) return rej(err) 176 if (err) return rej(err)
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index 30de4714c..452e43c8c 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -9,13 +9,13 @@ import {
9 ActivityPlaylistUrlObject, 9 ActivityPlaylistUrlObject,
10 ActivityTagObject, 10 ActivityTagObject,
11 ActivityUrlObject, 11 ActivityUrlObject,
12 ActivityVideoFileMetadataObject,
12 ActivityVideoUrlObject, 13 ActivityVideoUrlObject,
13 VideoState, 14 VideoState
14 ActivityVideoFileMetadataObject
15} from '../../../shared/index' 15} from '../../../shared/index'
16import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' 16import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
17import { VideoPrivacy } from '../../../shared/models/videos' 17import { VideoPrivacy } from '../../../shared/models/videos'
18import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' 18import { sanitizeAndCheckVideoTorrentObject, isAPVideoFileMetadataObject } from '../../helpers/custom-validators/activitypub/videos'
19import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' 19import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
20import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' 20import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
21import { logger } from '../../helpers/logger' 21import { logger } from '../../helpers/logger'
@@ -26,7 +26,8 @@ import {
26 P2P_MEDIA_LOADER_PEER_VERSION, 26 P2P_MEDIA_LOADER_PEER_VERSION,
27 PREVIEWS_SIZE, 27 PREVIEWS_SIZE,
28 REMOTE_SCHEME, 28 REMOTE_SCHEME,
29 STATIC_PATHS, THUMBNAILS_SIZE 29 STATIC_PATHS,
30 THUMBNAILS_SIZE
30} from '../../initializers/constants' 31} from '../../initializers/constants'
31import { TagModel } from '../../models/video/tag' 32import { TagModel } from '../../models/video/tag'
32import { VideoModel } from '../../models/video/video' 33import { VideoModel } from '../../models/video/video'
@@ -69,7 +70,8 @@ import {
69 MVideoAPWithoutCaption, 70 MVideoAPWithoutCaption,
70 MVideoFile, 71 MVideoFile,
71 MVideoFullLight, 72 MVideoFullLight,
72 MVideoId, MVideoImmutable, 73 MVideoId,
74 MVideoImmutable,
73 MVideoThumbnail 75 MVideoThumbnail
74} from '../../typings/models' 76} from '../../typings/models'
75import { MThumbnail } from '../../typings/models/video/thumbnail' 77import { MThumbnail } from '../../typings/models/video/thumbnail'
@@ -527,10 +529,6 @@ function isAPHashTagObject (url: any): url is ActivityHashTagObject {
527 return url && url.type === 'Hashtag' 529 return url && url.type === 'Hashtag'
528} 530}
529 531
530function isAPVideoFileMetadataObject (url: any): url is ActivityVideoFileMetadataObject {
531 return url && url.type === 'Link' && url.mediaType === 'application/json' && url.hasAttribute('rel') && url.rel.includes('metadata')
532}
533
534async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAccountLight, waitThumbnail = false) { 532async function createVideo (videoObject: VideoTorrentObject, channel: MChannelAccountLight, waitThumbnail = false) {
535 logger.debug('Adding remote video %s.', videoObject.id) 533 logger.debug('Adding remote video %s.', videoObject.id)
536 534
@@ -701,11 +699,11 @@ function videoFileActivityUrlToDBAttributes (
701 699
702 // Fetch associated metadata url, if any 700 // Fetch associated metadata url, if any
703 const metadata = urls.filter(isAPVideoFileMetadataObject) 701 const metadata = urls.filter(isAPVideoFileMetadataObject)
704 .find(u => 702 .find(u => {
705 u.height === fileUrl.height && 703 return u.height === fileUrl.height &&
706 u.fps === fileUrl.fps && 704 u.fps === fileUrl.fps &&
707 u.rel.includes(fileUrl.mediaType) 705 u.rel.includes(fileUrl.mediaType)
708 ) 706 })
709 707
710 const mediaType = fileUrl.mediaType 708 const mediaType = fileUrl.mediaType
711 const attribute = { 709 const attribute = {
diff --git a/server/lib/video-transcoding.ts b/server/lib/video-transcoding.ts
index 444b0d954..bde4c2633 100644
--- a/server/lib/video-transcoding.ts
+++ b/server/lib/video-transcoding.ts
@@ -237,12 +237,9 @@ async function onVideoFileTranscoding (video: MVideoWithFile, videoFile: MVideoF
237 237
238 await move(transcodingPath, outputPath) 238 await move(transcodingPath, outputPath)
239 239
240 const extractedVideo = extractVideo(video)
241
242 videoFile.size = stats.size 240 videoFile.size = stats.size
243 videoFile.fps = fps 241 videoFile.fps = fps
244 videoFile.metadata = metadata 242 videoFile.metadata = metadata
245 videoFile.metadataUrl = extractedVideo.getVideoFileMetadataUrl(videoFile, extractedVideo.getBaseUrls().baseUrlHttp)
246 243
247 await createTorrentAndSetInfoHash(video, videoFile) 244 await createTorrentAndSetInfoHash(video, videoFile)
248 245
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 029468004..201f0c0f1 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -30,18 +30,16 @@ import { MIMETYPES, MEMOIZE_LENGTH, MEMOIZE_TTL } from '../../initializers/const
30import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../typings/models/video/video-file' 30import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../typings/models/video/video-file'
31import { MStreamingPlaylistVideo, MVideo } from '@server/typings/models' 31import { MStreamingPlaylistVideo, MVideo } from '@server/typings/models'
32import * as memoizee from 'memoizee' 32import * as memoizee from 'memoizee'
33import validator from 'validator'
33 34
34export enum ScopeNames { 35export enum ScopeNames {
35 WITH_VIDEO = 'WITH_VIDEO', 36 WITH_VIDEO = 'WITH_VIDEO',
36 WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST',
37 WITH_METADATA = 'WITH_METADATA' 37 WITH_METADATA = 'WITH_METADATA'
38} 38}
39 39
40const METADATA_FIELDS = [ 'metadata', 'metadataUrl' ]
41
42@DefaultScope(() => ({ 40@DefaultScope(() => ({
43 attributes: { 41 attributes: {
44 exclude: [ METADATA_FIELDS[0] ] 42 exclude: [ 'metadata' ]
45 } 43 }
46})) 44}))
47@Scopes(() => ({ 45@Scopes(() => ({
@@ -53,35 +51,9 @@ const METADATA_FIELDS = [ 'metadata', 'metadataUrl' ]
53 } 51 }
54 ] 52 ]
55 }, 53 },
56 [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (videoIdOrUUID: string | number) => {
57 const where = (typeof videoIdOrUUID === 'number')
58 ? { id: videoIdOrUUID }
59 : { uuid: videoIdOrUUID }
60
61 return {
62 include: [
63 {
64 model: VideoModel.unscoped(),
65 required: false,
66 where
67 },
68 {
69 model: VideoStreamingPlaylistModel.unscoped(),
70 required: false,
71 include: [
72 {
73 model: VideoModel.unscoped(),
74 required: true,
75 where
76 }
77 ]
78 }
79 ]
80 }
81 },
82 [ScopeNames.WITH_METADATA]: { 54 [ScopeNames.WITH_METADATA]: {
83 attributes: { 55 attributes: {
84 include: METADATA_FIELDS 56 include: [ 'metadata' ]
85 } 57 }
86 } 58 }
87})) 59}))
@@ -223,10 +195,8 @@ export class VideoFileModel extends Model<VideoFileModel> {
223 195
224 static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) { 196 static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
225 const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID) 197 const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID)
226 return (videoFile?.Video.id === videoIdOrUUID) || 198
227 (videoFile?.Video.uuid === videoIdOrUUID) || 199 return !!videoFile
228 (videoFile?.VideoStreamingPlaylist?.Video?.id === videoIdOrUUID) ||
229 (videoFile?.VideoStreamingPlaylist?.Video?.uuid === videoIdOrUUID)
230 } 200 }
231 201
232 static loadWithMetadata (id: number) { 202 static loadWithMetadata (id: number) {
@@ -238,12 +208,41 @@ export class VideoFileModel extends Model<VideoFileModel> {
238 } 208 }
239 209
240 static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) { 210 static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) {
241 return VideoFileModel.scope({ 211 const whereVideo = validator.isUUID(videoIdOrUUID + '')
242 method: [ 212 ? { uuid: videoIdOrUUID }
243 ScopeNames.WITH_VIDEO_OR_PLAYLIST, 213 : { id: videoIdOrUUID }
244 videoIdOrUUID 214
215 const options = {
216 where: {
217 id
218 },
219 include: [
220 {
221 model: VideoModel.unscoped(),
222 required: false,
223 where: whereVideo
224 },
225 {
226 model: VideoStreamingPlaylistModel.unscoped(),
227 required: false,
228 include: [
229 {
230 model: VideoModel.unscoped(),
231 required: true,
232 where: whereVideo
233 }
234 ]
235 }
245 ] 236 ]
246 }).findByPk(id) 237 }
238
239 return VideoFileModel.findOne(options)
240 .then(file => {
241 // We used `required: false` so check we have at least a video or a streaming playlist
242 if (!file.Video && !file.VideoStreamingPlaylist) return null
243
244 return file
245 })
247 } 246 }
248 247
249 static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) { 248 static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) {
diff --git a/server/models/video/video-format-utils.ts b/server/models/video/video-format-utils.ts
index 21f0e0a68..365c9581e 100644
--- a/server/models/video/video-format-utils.ts
+++ b/server/models/video/video-format-utils.ts
@@ -181,6 +181,8 @@ function videoFilesModelToFormattedJSON (
181 baseUrlWs: string, 181 baseUrlWs: string,
182 videoFiles: MVideoFileRedundanciesOpt[] 182 videoFiles: MVideoFileRedundanciesOpt[]
183): VideoFile[] { 183): VideoFile[] {
184 const video = extractVideo(model)
185
184 return videoFiles 186 return videoFiles
185 .map(videoFile => { 187 .map(videoFile => {
186 return { 188 return {
@@ -195,7 +197,7 @@ function videoFilesModelToFormattedJSON (
195 torrentDownloadUrl: model.getTorrentDownloadUrl(videoFile, baseUrlHttp), 197 torrentDownloadUrl: model.getTorrentDownloadUrl(videoFile, baseUrlHttp),
196 fileUrl: model.getVideoFileUrl(videoFile, baseUrlHttp), 198 fileUrl: model.getVideoFileUrl(videoFile, baseUrlHttp),
197 fileDownloadUrl: model.getVideoFileDownloadUrl(videoFile, baseUrlHttp), 199 fileDownloadUrl: model.getVideoFileDownloadUrl(videoFile, baseUrlHttp),
198 metadataUrl: videoFile.metadataUrl // only send the metadataUrl and not the metadata over the wire 200 metadataUrl: video.getVideoFileMetadataUrl(videoFile, baseUrlHttp)
199 } as VideoFile 201 } as VideoFile
200 }) 202 })
201 .sort((a, b) => { 203 .sort((a, b) => {
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 5e4b7d44c..958a49e65 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -1849,7 +1849,8 @@ export class VideoModel extends Model<VideoModel> {
1849 1849
1850 getVideoFileMetadataUrl (videoFile: MVideoFile, baseUrlHttp: string) { 1850 getVideoFileMetadataUrl (videoFile: MVideoFile, baseUrlHttp: string) {
1851 const path = '/api/v1/videos/' 1851 const path = '/api/v1/videos/'
1852 return videoFile.metadata 1852
1853 return this.isOwned()
1853 ? baseUrlHttp + path + this.uuid + '/metadata/' + videoFile.id 1854 ? baseUrlHttp + path + this.uuid + '/metadata/' + videoFile.id
1854 : videoFile.metadataUrl 1855 : videoFile.metadataUrl
1855 } 1856 }
diff --git a/server/tests/api/videos/video-transcoder.ts b/server/tests/api/videos/video-transcoder.ts
index ce0dd14d5..13b3530b1 100644
--- a/server/tests/api/videos/video-transcoder.ts
+++ b/server/tests/api/videos/video-transcoder.ts
@@ -27,13 +27,14 @@ import {
27 root, 27 root,
28 ServerInfo, 28 ServerInfo,
29 setAccessTokensToServers, 29 setAccessTokensToServers,
30 uploadVideo, 30 uploadVideo, uploadVideoAndGetId,
31 waitJobs, 31 waitJobs,
32 webtorrentAdd 32 webtorrentAdd
33} from '../../../../shared/extra-utils' 33} from '../../../../shared/extra-utils'
34import { join } from 'path' 34import { join } from 'path'
35import { VIDEO_TRANSCODING_FPS } from '../../../../server/initializers/constants' 35import { VIDEO_TRANSCODING_FPS } from '../../../../server/initializers/constants'
36import { FfprobeData } from 'fluent-ffmpeg' 36import { FfprobeData } from 'fluent-ffmpeg'
37import { VideoFileMetadata } from '@shared/models/videos/video-file-metadata'
37 38
38const expect = chai.expect 39const expect = chai.expect
39 40
@@ -470,61 +471,56 @@ describe('Test video transcoding', function () {
470 it('Should provide valid ffprobe data', async function () { 471 it('Should provide valid ffprobe data', async function () {
471 this.timeout(160000) 472 this.timeout(160000)
472 473
473 const videoAttributes = { 474 const videoUUID = (await uploadVideoAndGetId({ server: servers[1], videoName: 'ffprobe data' })).uuid
474 name: 'my super name for server 1',
475 description: 'my super description for server 1',
476 fixture: 'video_short.webm'
477 }
478 await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributes)
479
480 await waitJobs(servers) 475 await waitJobs(servers)
481 476
482 const res = await getVideosList(servers[1].url) 477 {
478 const path = join(root(), 'test' + servers[1].internalServerNumber, 'videos', videoUUID + '-240.mp4')
479 const metadata = await getMetadataFromFile<VideoFileMetadata>(path)
483 480
484 const videoOnOrigin = res.body.data.find(v => v.name === videoAttributes.name) 481 // expected format properties
485 const res2 = await getVideo(servers[1].url, videoOnOrigin.id) 482 for (const p of [
486 const videoOnOriginDetails: VideoDetails = res2.body 483 'tags.encoder',
484 'format_long_name',
485 'size',
486 'bit_rate'
487 ]) {
488 expect(metadata.format).to.have.nested.property(p)
489 }
487 490
488 { 491 // expected stream properties
489 const path = join(root(), 'test' + servers[1].internalServerNumber, 'videos', videoOnOrigin.uuid + '-240.mp4')
490 const metadata = await getMetadataFromFile(path)
491 for (const p of [ 492 for (const p of [
492 // expected format properties 493 'codec_long_name',
493 'format.encoder', 494 'profile',
494 'format.format_long_name', 495 'width',
495 'format.size', 496 'height',
496 'format.bit_rate', 497 'display_aspect_ratio',
497 // expected stream properties 498 'avg_frame_rate',
498 'stream[0].codec_long_name', 499 'pix_fmt'
499 'stream[0].profile',
500 'stream[0].width',
501 'stream[0].height',
502 'stream[0].display_aspect_ratio',
503 'stream[0].avg_frame_rate',
504 'stream[0].pix_fmt'
505 ]) { 500 ]) {
506 expect(metadata).to.have.nested.property(p) 501 expect(metadata.streams[0]).to.have.nested.property(p)
507 } 502 }
503
508 expect(metadata).to.not.have.nested.property('format.filename') 504 expect(metadata).to.not.have.nested.property('format.filename')
509 } 505 }
510 506
511 for (const server of servers) { 507 for (const server of servers) {
512 const res = await getVideosList(server.url) 508 const res2 = await getVideo(server.url, videoUUID)
513 509 const videoDetails: VideoDetails = res2.body
514 const video = res.body.data.find(v => v.name === videoAttributes.name)
515 const res2 = await getVideo(server.url, video.id)
516 const videoDetails = res2.body
517 510
518 const videoFiles = videoDetails.files 511 const videoFiles = videoDetails.files
519 for (const [ index, file ] of videoFiles.entries()) { 512 .concat(videoDetails.streamingPlaylists[0].files)
513 expect(videoFiles).to.have.lengthOf(8)
514
515 for (const file of videoFiles) {
520 expect(file.metadata).to.be.undefined 516 expect(file.metadata).to.be.undefined
517 expect(file.metadataUrl).to.exist
521 expect(file.metadataUrl).to.contain(servers[1].url) 518 expect(file.metadataUrl).to.contain(servers[1].url)
522 expect(file.metadataUrl).to.contain(videoOnOrigin.uuid) 519 expect(file.metadataUrl).to.contain(videoUUID)
523 520
524 const res3 = await getVideoFileMetadataUrl(file.metadataUrl) 521 const res3 = await getVideoFileMetadataUrl(file.metadataUrl)
525 const metadata: FfprobeData = res3.body 522 const metadata: FfprobeData = res3.body
526 expect(metadata).to.have.nested.property('format.size') 523 expect(metadata).to.have.nested.property('format.size')
527 expect(metadata.format.size).to.equal(videoOnOriginDetails.files[index].metadata.format.size)
528 } 524 }
529 } 525 }
530 }) 526 })