aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/initializers/constants.ts3
-rw-r--r--server/lib/job-queue/handlers/video-import.ts76
-rw-r--r--server/lib/moderation.ts24
-rw-r--r--server/middlewares/validators/videos/video-imports.ts43
-rw-r--r--server/tests/fixtures/peertube-plugin-test/main.js42
-rw-r--r--server/tests/plugins/filter-hooks.ts88
6 files changed, 243 insertions, 33 deletions
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index e5cac64d4..676d9804b 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -372,7 +372,8 @@ const VIDEO_STATES = {
372const VIDEO_IMPORT_STATES = { 372const VIDEO_IMPORT_STATES = {
373 [VideoImportState.FAILED]: 'Failed', 373 [VideoImportState.FAILED]: 'Failed',
374 [VideoImportState.PENDING]: 'Pending', 374 [VideoImportState.PENDING]: 'Pending',
375 [VideoImportState.SUCCESS]: 'Success' 375 [VideoImportState.SUCCESS]: 'Success',
376 [VideoImportState.REJECTED]: 'Rejected'
376} 377}
377 378
378const VIDEO_ABUSE_STATES = { 379const VIDEO_ABUSE_STATES = {
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index ad549c6fc..a197ef629 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -1,27 +1,36 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { logger } from '../../../helpers/logger' 2import { move, remove, stat } from 'fs-extra'
3import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl' 3import { extname } from 'path'
4import { VideoImportModel } from '../../../models/video/video-import' 4import { addOptimizeOrMergeAudioJob } from '@server/helpers/video'
5import { isPostImportVideoAccepted } from '@server/lib/moderation'
6import { Hooks } from '@server/lib/plugins/hooks'
7import { getVideoFilePath } from '@server/lib/video-paths'
8import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import'
9import {
10 VideoImportPayload,
11 VideoImportTorrentPayload,
12 VideoImportTorrentPayloadType,
13 VideoImportYoutubeDLPayload,
14 VideoImportYoutubeDLPayloadType,
15 VideoState
16} from '../../../../shared'
5import { VideoImportState } from '../../../../shared/models/videos' 17import { VideoImportState } from '../../../../shared/models/videos'
18import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
6import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils' 19import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
7import { extname } from 'path' 20import { logger } from '../../../helpers/logger'
8import { VideoFileModel } from '../../../models/video/video-file'
9import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
10import { VideoImportPayload, VideoImportTorrentPayload, VideoImportYoutubeDLPayload, VideoState } from '../../../../shared'
11import { federateVideoIfNeeded } from '../../activitypub/videos'
12import { VideoModel } from '../../../models/video/video'
13import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
14import { getSecureTorrentName } from '../../../helpers/utils' 21import { getSecureTorrentName } from '../../../helpers/utils'
15import { move, remove, stat } from 'fs-extra' 22import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
16import { Notifier } from '../../notifier' 23import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl'
17import { CONFIG } from '../../../initializers/config' 24import { CONFIG } from '../../../initializers/config'
25import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
18import { sequelizeTypescript } from '../../../initializers/database' 26import { sequelizeTypescript } from '../../../initializers/database'
19import { generateVideoMiniature } from '../../thumbnail' 27import { VideoModel } from '../../../models/video/video'
20import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type' 28import { VideoFileModel } from '../../../models/video/video-file'
29import { VideoImportModel } from '../../../models/video/video-import'
21import { MThumbnail } from '../../../typings/models/video/thumbnail' 30import { MThumbnail } from '../../../typings/models/video/thumbnail'
22import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import' 31import { federateVideoIfNeeded } from '../../activitypub/videos'
23import { getVideoFilePath } from '@server/lib/video-paths' 32import { Notifier } from '../../notifier'
24import { addOptimizeOrMergeAudioJob } from '@server/helpers/video' 33import { generateVideoMiniature } from '../../thumbnail'
25 34
26async function processVideoImport (job: Bull.Job) { 35async function processVideoImport (job: Bull.Job) {
27 const payload = job.data as VideoImportPayload 36 const payload = job.data as VideoImportPayload
@@ -44,6 +53,7 @@ async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentP
44 const videoImport = await getVideoImportOrDie(payload.videoImportId) 53 const videoImport = await getVideoImportOrDie(payload.videoImportId)
45 54
46 const options = { 55 const options = {
56 type: payload.type,
47 videoImportId: payload.videoImportId, 57 videoImportId: payload.videoImportId,
48 58
49 generateThumbnail: true, 59 generateThumbnail: true,
@@ -61,6 +71,7 @@ async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutub
61 71
62 const videoImport = await getVideoImportOrDie(payload.videoImportId) 72 const videoImport = await getVideoImportOrDie(payload.videoImportId)
63 const options = { 73 const options = {
74 type: payload.type,
64 videoImportId: videoImport.id, 75 videoImportId: videoImport.id,
65 76
66 generateThumbnail: payload.generateThumbnail, 77 generateThumbnail: payload.generateThumbnail,
@@ -80,6 +91,7 @@ async function getVideoImportOrDie (videoImportId: number) {
80} 91}
81 92
82type ProcessFileOptions = { 93type ProcessFileOptions = {
94 type: VideoImportYoutubeDLPayloadType | VideoImportTorrentPayloadType
83 videoImportId: number 95 videoImportId: number
84 96
85 generateThumbnail: boolean 97 generateThumbnail: boolean
@@ -105,7 +117,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
105 const fps = await getVideoFileFPS(tempVideoPath) 117 const fps = await getVideoFileFPS(tempVideoPath)
106 const duration = await getDurationFromVideoFile(tempVideoPath) 118 const duration = await getDurationFromVideoFile(tempVideoPath)
107 119
108 // Create video file object in database 120 // Prepare video file object for creation in database
109 const videoFileData = { 121 const videoFileData = {
110 extname: extname(tempVideoPath), 122 extname: extname(tempVideoPath),
111 resolution: videoFileResolution, 123 resolution: videoFileResolution,
@@ -115,6 +127,30 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
115 } 127 }
116 videoFile = new VideoFileModel(videoFileData) 128 videoFile = new VideoFileModel(videoFileData)
117 129
130 const hookName = options.type === 'youtube-dl'
131 ? 'filter:api.video.post-import-url.accept.result'
132 : 'filter:api.video.post-import-torrent.accept.result'
133
134 // Check we accept this video
135 const acceptParameters = {
136 videoImport,
137 video: videoImport.Video,
138 videoFilePath: tempVideoPath,
139 videoFile,
140 user: videoImport.User
141 }
142 const acceptedResult = await Hooks.wrapFun(isPostImportVideoAccepted, acceptParameters, hookName)
143
144 if (acceptedResult.accepted !== true) {
145 logger.info('Refused imported video.', { acceptedResult, acceptParameters })
146
147 videoImport.state = VideoImportState.REJECTED
148 await videoImport.save()
149
150 throw new Error(acceptedResult.errorMessage)
151 }
152
153 // Video is accepted, resuming preparation
118 const videoWithFiles = Object.assign(videoImport.Video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] }) 154 const videoWithFiles = Object.assign(videoImport.Video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] })
119 // To clean files if the import fails 155 // To clean files if the import fails
120 const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles }) 156 const videoImportWithFiles: MVideoImportDefaultFiles = Object.assign(videoImport, { Video: videoWithFiles })
@@ -194,7 +230,9 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
194 } 230 }
195 231
196 videoImport.error = err.message 232 videoImport.error = err.message
197 videoImport.state = VideoImportState.FAILED 233 if (videoImport.state !== VideoImportState.REJECTED) {
234 videoImport.state = VideoImportState.FAILED
235 }
198 await videoImport.save() 236 await videoImport.save()
199 237
200 Notifier.Instance.notifyOnFinishedVideoImport(videoImport, false) 238 Notifier.Instance.notifyOnFinishedVideoImport(videoImport, false)
diff --git a/server/lib/moderation.ts b/server/lib/moderation.ts
index 55f7a985d..4afebb32a 100644
--- a/server/lib/moderation.ts
+++ b/server/lib/moderation.ts
@@ -1,12 +1,15 @@
1import { VideoModel } from '../models/video/video' 1import { VideoModel } from '../models/video/video'
2import { VideoCommentModel } from '../models/video/video-comment' 2import { VideoCommentModel } from '../models/video/video-comment'
3import { VideoCommentCreate } from '../../shared/models/videos/video-comment.model' 3import { VideoCommentCreate } from '../../shared/models/videos/video-comment.model'
4import { VideoCreate } from '../../shared/models/videos' 4import { VideoCreate, VideoImportCreate } from '../../shared/models/videos'
5import { UserModel } from '../models/account/user' 5import { UserModel } from '../models/account/user'
6import { VideoTorrentObject } from '../../shared/models/activitypub/objects' 6import { VideoTorrentObject } from '../../shared/models/activitypub/objects'
7import { ActivityCreate } from '../../shared/models/activitypub' 7import { ActivityCreate } from '../../shared/models/activitypub'
8import { ActorModel } from '../models/activitypub/actor' 8import { ActorModel } from '../models/activitypub/actor'
9import { VideoCommentObject } from '../../shared/models/activitypub/objects/video-comment-object' 9import { VideoCommentObject } from '../../shared/models/activitypub/objects/video-comment-object'
10import { VideoFileModel } from '@server/models/video/video-file'
11import { PathLike } from 'fs-extra'
12import { MUser } from '@server/typings/models'
10 13
11export type AcceptResult = { 14export type AcceptResult = {
12 accepted: boolean 15 accepted: boolean
@@ -55,10 +58,27 @@ function isRemoteVideoCommentAccepted (_object: {
55 return { accepted: true } 58 return { accepted: true }
56} 59}
57 60
61function isPreImportVideoAccepted (object: {
62 videoImportBody: VideoImportCreate
63 user: MUser
64}): AcceptResult {
65 return { accepted: true }
66}
67
68function isPostImportVideoAccepted (object: {
69 videoFilePath: PathLike
70 videoFile: VideoFileModel
71 user: MUser
72}): AcceptResult {
73 return { accepted: true }
74}
75
58export { 76export {
59 isLocalVideoAccepted, 77 isLocalVideoAccepted,
60 isLocalVideoThreadAccepted, 78 isLocalVideoThreadAccepted,
61 isRemoteVideoAccepted, 79 isRemoteVideoAccepted,
62 isRemoteVideoCommentAccepted, 80 isRemoteVideoCommentAccepted,
63 isLocalVideoCommentReplyAccepted 81 isLocalVideoCommentReplyAccepted,
82 isPreImportVideoAccepted,
83 isPostImportVideoAccepted
64} 84}
diff --git a/server/middlewares/validators/videos/video-imports.ts b/server/middlewares/validators/videos/video-imports.ts
index 5dc5db533..e3d900a9e 100644
--- a/server/middlewares/validators/videos/video-imports.ts
+++ b/server/middlewares/validators/videos/video-imports.ts
@@ -1,15 +1,18 @@
1import * as express from 'express' 1import * as express from 'express'
2import { body } from 'express-validator' 2import { body } from 'express-validator'
3import { isPreImportVideoAccepted } from '@server/lib/moderation'
4import { Hooks } from '@server/lib/plugins/hooks'
5import { VideoImportCreate } from '@shared/models/videos/import/video-import-create.model'
3import { isIdValid, toIntOrNull } from '../../../helpers/custom-validators/misc' 6import { isIdValid, toIntOrNull } from '../../../helpers/custom-validators/misc'
4import { logger } from '../../../helpers/logger'
5import { areValidationErrors } from '../utils'
6import { getCommonVideoEditAttributes } from './videos'
7import { isVideoImportTargetUrlValid, isVideoImportTorrentFile } from '../../../helpers/custom-validators/video-imports' 7import { isVideoImportTargetUrlValid, isVideoImportTorrentFile } from '../../../helpers/custom-validators/video-imports'
8import { cleanUpReqFiles } from '../../../helpers/express-utils'
9import { isVideoMagnetUriValid, isVideoNameValid } from '../../../helpers/custom-validators/videos' 8import { isVideoMagnetUriValid, isVideoNameValid } from '../../../helpers/custom-validators/videos'
9import { cleanUpReqFiles } from '../../../helpers/express-utils'
10import { logger } from '../../../helpers/logger'
11import { doesVideoChannelOfAccountExist } from '../../../helpers/middlewares'
10import { CONFIG } from '../../../initializers/config' 12import { CONFIG } from '../../../initializers/config'
11import { CONSTRAINTS_FIELDS } from '../../../initializers/constants' 13import { CONSTRAINTS_FIELDS } from '../../../initializers/constants'
12import { doesVideoChannelOfAccountExist } from '../../../helpers/middlewares' 14import { areValidationErrors } from '../utils'
15import { getCommonVideoEditAttributes } from './videos'
13 16
14const videoImportAddValidator = getCommonVideoEditAttributes().concat([ 17const videoImportAddValidator = getCommonVideoEditAttributes().concat([
15 body('channelId') 18 body('channelId')
@@ -64,6 +67,8 @@ const videoImportAddValidator = getCommonVideoEditAttributes().concat([
64 .end() 67 .end()
65 } 68 }
66 69
70 if (!await isImportAccepted(req, res)) return cleanUpReqFiles(req)
71
67 return next() 72 return next()
68 } 73 }
69]) 74])
@@ -75,3 +80,31 @@ export {
75} 80}
76 81
77// --------------------------------------------------------------------------- 82// ---------------------------------------------------------------------------
83
84async function isImportAccepted (req: express.Request, res: express.Response) {
85 const body: VideoImportCreate = req.body
86 const hookName = body.targetUrl
87 ? 'filter:api.video.pre-import-url.accept.result'
88 : 'filter:api.video.pre-import-torrent.accept.result'
89
90 // Check we accept this video
91 const acceptParameters = {
92 videoImportBody: body,
93 user: res.locals.oauth.token.User
94 }
95 const acceptedResult = await Hooks.wrapFun(
96 isPreImportVideoAccepted,
97 acceptParameters,
98 hookName
99 )
100
101 if (!acceptedResult || acceptedResult.accepted !== true) {
102 logger.info('Refused to import video.', { acceptedResult, acceptParameters })
103 res.status(403)
104 .json({ error: acceptedResult.errorMessage || 'Refused to import video' })
105
106 return false
107 }
108
109 return true
110}
diff --git a/server/tests/fixtures/peertube-plugin-test/main.js b/server/tests/fixtures/peertube-plugin-test/main.js
index 69796ab07..a45e98fb5 100644
--- a/server/tests/fixtures/peertube-plugin-test/main.js
+++ b/server/tests/fixtures/peertube-plugin-test/main.js
@@ -50,7 +50,47 @@ async function register ({ registerHook, registerSetting, settingsManager, stora
50 target: 'filter:api.video.upload.accept.result', 50 target: 'filter:api.video.upload.accept.result',
51 handler: ({ accepted }, { videoBody }) => { 51 handler: ({ accepted }, { videoBody }) => {
52 if (!accepted) return { accepted: false } 52 if (!accepted) return { accepted: false }
53 if (videoBody.name.indexOf('bad word') !== -1) return { accepted: false, errorMessage: 'bad word '} 53 if (videoBody.name.indexOf('bad word') !== -1) return { accepted: false, errorMessage: 'bad word' }
54
55 return { accepted: true }
56 }
57 })
58
59 registerHook({
60 target: 'filter:api.video.pre-import-url.accept.result',
61 handler: ({ accepted }, { videoImportBody }) => {
62 if (!accepted) return { accepted: false }
63 if (videoImportBody.targetUrl.includes('bad')) return { accepted: false, errorMessage: 'bad target url' }
64
65 return { accepted: true }
66 }
67 })
68
69 registerHook({
70 target: 'filter:api.video.pre-import-torrent.accept.result',
71 handler: ({ accepted }, { videoImportBody }) => {
72 if (!accepted) return { accepted: false }
73 if (videoImportBody.name.includes('bad torrent')) return { accepted: false, errorMessage: 'bad torrent' }
74
75 return { accepted: true }
76 }
77 })
78
79 registerHook({
80 target: 'filter:api.video.post-import-url.accept.result',
81 handler: ({ accepted }, { video }) => {
82 if (!accepted) return { accepted: false }
83 if (video.name.includes('bad word')) return { accepted: false, errorMessage: 'bad word' }
84
85 return { accepted: true }
86 }
87 })
88
89 registerHook({
90 target: 'filter:api.video.post-import-torrent.accept.result',
91 handler: ({ accepted }, { video }) => {
92 if (!accepted) return { accepted: false }
93 if (video.name.includes('bad word')) return { accepted: false, errorMessage: 'bad word' }
54 94
55 return { accepted: true } 95 return { accepted: true }
56 } 96 }
diff --git a/server/tests/plugins/filter-hooks.ts b/server/tests/plugins/filter-hooks.ts
index 6c1fd40ba..41242318e 100644
--- a/server/tests/plugins/filter-hooks.ts
+++ b/server/tests/plugins/filter-hooks.ts
@@ -1,8 +1,8 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import * as chai from 'chai'
4import 'mocha' 3import 'mocha'
5import { cleanupTests, flushAndRunMultipleServers, ServerInfo } from '../../../shared/extra-utils/server/servers' 4import * as chai from 'chai'
5import { ServerConfig } from '@shared/models'
6import { 6import {
7 addVideoCommentReply, 7 addVideoCommentReply,
8 addVideoCommentThread, 8 addVideoCommentThread,
@@ -23,10 +23,10 @@ import {
23 uploadVideo, 23 uploadVideo,
24 waitJobs 24 waitJobs
25} from '../../../shared/extra-utils' 25} from '../../../shared/extra-utils'
26import { cleanupTests, flushAndRunMultipleServers, ServerInfo } from '../../../shared/extra-utils/server/servers'
27import { getMyVideoImports, getYoutubeVideoUrl, importVideo } from '../../../shared/extra-utils/videos/video-imports'
28import { VideoDetails, VideoImport, VideoImportState, VideoPrivacy } from '../../../shared/models/videos'
26import { VideoCommentThreadTree } from '../../../shared/models/videos/video-comment.model' 29import { VideoCommentThreadTree } from '../../../shared/models/videos/video-comment.model'
27import { VideoDetails } from '../../../shared/models/videos'
28import { getYoutubeVideoUrl, importVideo } from '../../../shared/extra-utils/videos/video-imports'
29import { ServerConfig } from '@shared/models'
30 30
31const expect = chai.expect 31const expect = chai.expect
32 32
@@ -87,6 +87,84 @@ describe('Test plugin filter hooks', function () {
87 await uploadVideo(servers[0].url, servers[0].accessToken, { name: 'video with bad word' }, 403) 87 await uploadVideo(servers[0].url, servers[0].accessToken, { name: 'video with bad word' }, 403)
88 }) 88 })
89 89
90 it('Should run filter:api.video.pre-import-url.accept.result', async function () {
91 const baseAttributes = {
92 name: 'normal title',
93 privacy: VideoPrivacy.PUBLIC,
94 channelId: servers[0].videoChannel.id,
95 targetUrl: getYoutubeVideoUrl() + 'bad'
96 }
97 await importVideo(servers[0].url, servers[0].accessToken, baseAttributes, 403)
98 })
99
100 it('Should run filter:api.video.pre-import-torrent.accept.result', async function () {
101 const baseAttributes = {
102 name: 'bad torrent',
103 privacy: VideoPrivacy.PUBLIC,
104 channelId: servers[0].videoChannel.id,
105 torrentfile: 'video-720p.torrent' as any
106 }
107 await importVideo(servers[0].url, servers[0].accessToken, baseAttributes, 403)
108 })
109
110 it('Should run filter:api.video.post-import-url.accept.result', async function () {
111 this.timeout(60000)
112
113 let videoImportId: number
114
115 {
116 const baseAttributes = {
117 name: 'title with bad word',
118 privacy: VideoPrivacy.PUBLIC,
119 channelId: servers[0].videoChannel.id,
120 targetUrl: getYoutubeVideoUrl()
121 }
122 const res = await importVideo(servers[0].url, servers[0].accessToken, baseAttributes)
123 videoImportId = res.body.id
124 }
125
126 await waitJobs(servers)
127
128 {
129 const res = await getMyVideoImports(servers[0].url, servers[0].accessToken)
130 const videoImports = res.body.data as VideoImport[]
131
132 const videoImport = videoImports.find(i => i.id === videoImportId)
133
134 expect(videoImport.state.id).to.equal(VideoImportState.REJECTED)
135 expect(videoImport.state.label).to.equal('Rejected')
136 }
137 })
138
139 it('Should run filter:api.video.post-import-torrent.accept.result', async function () {
140 this.timeout(60000)
141
142 let videoImportId: number
143
144 {
145 const baseAttributes = {
146 name: 'title with bad word',
147 privacy: VideoPrivacy.PUBLIC,
148 channelId: servers[0].videoChannel.id,
149 torrentfile: 'video-720p.torrent' as any
150 }
151 const res = await importVideo(servers[0].url, servers[0].accessToken, baseAttributes)
152 videoImportId = res.body.id
153 }
154
155 await waitJobs(servers)
156
157 {
158 const res = await getMyVideoImports(servers[0].url, servers[0].accessToken)
159 const videoImports = res.body.data as VideoImport[]
160
161 const videoImport = videoImports.find(i => i.id === videoImportId)
162
163 expect(videoImport.state.id).to.equal(VideoImportState.REJECTED)
164 expect(videoImport.state.label).to.equal('Rejected')
165 }
166 })
167
90 it('Should run filter:api.video-thread.create.accept.result', async function () { 168 it('Should run filter:api.video-thread.create.accept.result', async function () {
91 await addVideoCommentThread(servers[0].url, servers[0].accessToken, videoUUID, 'comment with bad word', 403) 169 await addVideoCommentThread(servers[0].url, servers[0].accessToken, videoUUID, 'comment with bad word', 403)
92 }) 170 })