From a84b8fa5cf6e4cafb841af3db9bdfcc9531c09a4 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 7 Aug 2018 10:07:53 +0200 Subject: [PATCH] Add import.video.torrent configuration --- .../edit-custom-config.component.html | 5 ++ .../edit-custom-config.component.ts | 7 +- client/src/app/core/server/server.service.ts | 3 + .../+video-edit/video-add.component.scss | 1 + .../videos/+video-edit/video-add.component.ts | 2 +- config/default.yaml | 2 + config/production.yaml.example | 2 + config/test.yaml | 2 + server/controllers/api/config.ts | 8 +- server/controllers/api/users.ts | 2 +- server/controllers/api/videos/import.ts | 10 ++- server/initializers/constants.ts | 5 +- .../migrations/0245-import-magnet.ts | 42 ---------- server/lib/job-queue/handlers/video-import.ts | 9 +- server/middlewares/validators/config.ts | 1 + .../middlewares/validators/video-imports.ts | 15 +++- server/models/account/user.ts | 2 +- server/models/video/video-import.ts | 83 ++++++++----------- server/tests/api/check-params/config.ts | 3 + server/tests/api/server/config.ts | 5 ++ server/tests/utils/server/config.ts | 3 + shared/models/server/custom-config.model.ts | 3 + shared/models/server/server-config.model.ts | 3 + 23 files changed, 112 insertions(+), 106 deletions(-) delete mode 100644 server/initializers/migrations/0245-import-magnet.ts diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html index 13b43306b..0a032df12 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html +++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html @@ -108,6 +108,11 @@ i18n-labelText labelText="Video import with HTTP enabled" > + +
Administrator
diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts index bc5ce6e5d..fd6784415 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts @@ -72,6 +72,7 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit { signupEnabled: null, signupLimit: this.customConfigValidatorsService.SIGNUP_LIMIT, importVideosHttpEnabled: null, + importVideosTorrentEnabled: null, adminEmail: this.customConfigValidatorsService.ADMIN_EMAIL, userVideoQuota: this.userValidatorsService.USER_VIDEO_QUOTA, transcodingThreads: this.customConfigValidatorsService.TRANSCODING_THREADS, @@ -189,6 +190,9 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit { videos: { http: { enabled: this.form.value['importVideosHttpEnabled'] + }, + torrent: { + enabled: this.form.value['importVideosTorrentEnabled'] } } } @@ -231,7 +235,8 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit { transcodingEnabled: this.customConfig.transcoding.enabled, customizationJavascript: this.customConfig.instance.customizations.javascript, customizationCSS: this.customConfig.instance.customizations.css, - importVideosHttpEnabled: this.customConfig.import.videos.http.enabled + importVideosHttpEnabled: this.customConfig.import.videos.http.enabled, + importVideosTorrentEnabled: this.customConfig.import.videos.torrent.enabled } for (const resolution of this.resolutions) { diff --git a/client/src/app/core/server/server.service.ts b/client/src/app/core/server/server.service.ts index ab317f0aa..52b50cbe8 100644 --- a/client/src/app/core/server/server.service.ts +++ b/client/src/app/core/server/server.service.ts @@ -73,6 +73,9 @@ export class ServerService { videos: { http: { enabled: false + }, + torrent: { + enabled: false } } } diff --git a/client/src/app/videos/+video-edit/video-add.component.scss b/client/src/app/videos/+video-edit/video-add.component.scss index 443361f50..c1f96cc37 100644 --- a/client/src/app/videos/+video-edit/video-add.component.scss +++ b/client/src/app/videos/+video-edit/video-add.component.scss @@ -50,6 +50,7 @@ $background-color: #F7F7F7; border-radius: 3px; width: 100%; min-height: 440px; + padding-bottom: 20px; display: flex; justify-content: center; align-items: center; diff --git a/client/src/app/videos/+video-edit/video-add.component.ts b/client/src/app/videos/+video-edit/video-add.component.ts index 7d360598d..1a9247dbe 100644 --- a/client/src/app/videos/+video-edit/video-add.component.ts +++ b/client/src/app/videos/+video-edit/video-add.component.ts @@ -40,6 +40,6 @@ export class VideoAddComponent implements CanComponentDeactivate { } isVideoImportTorrentEnabled () { - return this.serverService.getConfig().import.videos.http.enabled + return this.serverService.getConfig().import.videos.torrent.enabled } } diff --git a/config/default.yaml b/config/default.yaml index 5fa7e5945..60da192b4 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -97,6 +97,8 @@ import: videos: http: # Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html enabled: false + torrent: # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file) + enabled: false instance: name: 'PeerTube' diff --git a/config/production.yaml.example b/config/production.yaml.example index 635a41e9e..9e8b57829 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -111,6 +111,8 @@ import: videos: http: # Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html enabled: false + torrent: # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file) + enabled: false # Instance settings instance: diff --git a/config/test.yaml b/config/test.yaml index 3c51785f2..879b6bdd4 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -44,6 +44,8 @@ import: videos: http: enabled: true + torrent: + enabled: true instance: default_nsfw_policy: 'display' \ No newline at end of file diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts index 950a1498e..6f05c33db 100644 --- a/server/controllers/api/config.ts +++ b/server/controllers/api/config.ts @@ -9,7 +9,7 @@ import { CONFIG, CONSTRAINTS_FIELDS, reloadConfig } from '../../initializers' import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../middlewares' import { customConfigUpdateValidator } from '../../middlewares/validators/config' import { ClientHtml } from '../../lib/client-html' -import { CustomConfigAuditView, auditLoggerFactory } from '../../helpers/audit-logger' +import { auditLoggerFactory, CustomConfigAuditView } from '../../helpers/audit-logger' const packageJSON = require('../../../../package.json') const configRouter = express.Router() @@ -69,6 +69,9 @@ async function getConfig (req: express.Request, res: express.Response, next: exp videos: { http: { enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED + }, + torrent: { + enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED } } }, @@ -237,6 +240,9 @@ function customConfig (): CustomConfig { videos: { http: { enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED + }, + torrent: { + enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED } } } diff --git a/server/controllers/api/users.ts b/server/controllers/api/users.ts index 879ba3f91..36bf0e0fe 100644 --- a/server/controllers/api/users.ts +++ b/server/controllers/api/users.ts @@ -196,7 +196,7 @@ async function getUserVideos (req: express.Request, res: express.Response, next: async function getUserVideoImports (req: express.Request, res: express.Response, next: express.NextFunction) { const user = res.locals.oauth.token.User as UserModel const resultList = await VideoImportModel.listUserVideoImportsForApi( - user.Account.id, + user.id, req.query.start as number, req.query.count as number, req.query.sort diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index df151e79d..94dafcdbd 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts @@ -61,12 +61,13 @@ export { function addVideoImport (req: express.Request, res: express.Response) { if (req.body.targetUrl) return addYoutubeDLImport(req, res) - const file = req.files['torrentfile'][0] + const file = req.files && req.files['torrentfile'] ? req.files['torrentfile'][0] : undefined if (req.body.magnetUri || file) return addTorrentImport(req, res, file) } async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { const body: VideoImportCreate = req.body + const user = res.locals.oauth.token.User let videoName: string let torrentName: string @@ -100,7 +101,8 @@ async function addTorrentImport (req: express.Request, res: express.Response, to const videoImportAttributes = { magnetUri, torrentName, - state: VideoImportState.PENDING + state: VideoImportState.PENDING, + userId: user.id } const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes) @@ -120,6 +122,7 @@ async function addTorrentImport (req: express.Request, res: express.Response, to async function addYoutubeDLImport (req: express.Request, res: express.Response) { const body: VideoImportCreate = req.body const targetUrl = body.targetUrl + const user = res.locals.oauth.token.User let youtubeDLInfo: YoutubeDLInfo try { @@ -140,7 +143,8 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response) const tags = body.tags || youtubeDLInfo.tags const videoImportAttributes = { targetUrl, - state: VideoImportState.PENDING + state: VideoImportState.PENDING, + userId: user.id } const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes) diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index cf7cd3d74..80eb3f1e7 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -15,7 +15,7 @@ let config: IConfig = require('config') // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 245 +const LAST_MIGRATION_VERSION = 240 // --------------------------------------------------------------------------- @@ -211,6 +211,9 @@ const CONFIG = { VIDEOS: { HTTP: { get ENABLED () { return config.get('import.videos.http.enabled') } + }, + TORRENT: { + get ENABLED () { return config.get('import.videos.torrent.enabled') } } } }, diff --git a/server/initializers/migrations/0245-import-magnet.ts b/server/initializers/migrations/0245-import-magnet.ts deleted file mode 100644 index 87603b006..000000000 --- a/server/initializers/migrations/0245-import-magnet.ts +++ /dev/null @@ -1,42 +0,0 @@ -import * as Sequelize from 'sequelize' -import { Migration } from '../../models/migrations' -import { CONSTRAINTS_FIELDS } from '../index' - -async function up (utils: { - transaction: Sequelize.Transaction - queryInterface: Sequelize.QueryInterface - sequelize: Sequelize.Sequelize -}): Promise { - { - const data = { - type: Sequelize.STRING, - allowNull: true, - defaultValue: null - } as Migration.String - await utils.queryInterface.changeColumn('videoImport', 'targetUrl', data) - } - - { - const data = { - type: Sequelize.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max), - allowNull: true, - defaultValue: null - } - await utils.queryInterface.addColumn('videoImport', 'magnetUri', data) - } - - { - const data = { - type: Sequelize.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_NAME.max), - allowNull: true, - defaultValue: null - } - await utils.queryInterface.addColumn('videoImport', 'torrentName', data) - } -} - -function down (options) { - throw new Error('Not implemented.') -} - -export { up, down } diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index fd61aecad..28a03d19e 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -114,16 +114,21 @@ async function processFile (downloader: () => Promise, videoImport: Vide tempVideoPath = await downloader() // Get information about this video + const { size } = await statPromise(tempVideoPath) + const isAble = await videoImport.User.isAbleToUploadVideo({ size }) + if (isAble === false) { + throw new Error('The user video quota is exceeded with this video to import.') + } + const { videoFileResolution } = await getVideoFileResolution(tempVideoPath) const fps = await getVideoFileFPS(tempVideoPath) - const stats = await statPromise(tempVideoPath) const duration = await getDurationFromVideoFile(tempVideoPath) // Create video file object in database const videoFileData = { extname: extname(tempVideoPath), resolution: videoFileResolution, - size: stats.size, + size, fps, videoId: videoImport.videoId } diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts index 9d303eee2..f3f257d57 100644 --- a/server/middlewares/validators/config.ts +++ b/server/middlewares/validators/config.ts @@ -25,6 +25,7 @@ const customConfigUpdateValidator = [ body('transcoding.resolutions.720p').isBoolean().withMessage('Should have a valid transcoding 720p resolution enabled boolean'), body('transcoding.resolutions.1080p').isBoolean().withMessage('Should have a valid transcoding 1080p resolution enabled boolean'), body('import.videos.http.enabled').isBoolean().withMessage('Should have a valid import video http enabled boolean'), + body('import.videos.torrent.enabled').isBoolean().withMessage('Should have a valid import video torrent enabled boolean'), async (req: express.Request, res: express.Response, next: express.NextFunction) => { logger.debug('Checking customConfigUpdateValidator parameters', { parameters: req.body }) diff --git a/server/middlewares/validators/video-imports.ts b/server/middlewares/validators/video-imports.ts index c03cf2e4d..9ac739101 100644 --- a/server/middlewares/validators/video-imports.ts +++ b/server/middlewares/validators/video-imports.ts @@ -33,21 +33,28 @@ const videoImportAddValidator = getCommonVideoAttributes().concat([ logger.debug('Checking videoImportAddValidator parameters', { parameters: req.body }) const user = res.locals.oauth.token.User + const torrentFile = req.files && req.files['torrentfile'] ? req.files['torrentfile'][0] : undefined if (areValidationErrors(req, res)) return cleanUpReqFiles(req) - if (CONFIG.IMPORT.VIDEOS.HTTP.ENABLED !== true) { + if (req.body.targetUrl && CONFIG.IMPORT.VIDEOS.HTTP.ENABLED !== true) { cleanUpReqFiles(req) return res.status(409) - .json({ error: 'Import is not enabled on this instance.' }) + .json({ error: 'HTTP import is not enabled on this instance.' }) .end() } + if (CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED !== true && (req.body.magnetUri || torrentFile)) { + cleanUpReqFiles(req) + return res.status(409) + .json({ error: 'Torrent/magnet URI import is not enabled on this instance.' }) + .end() + } + if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) // Check we have at least 1 required param - const file = req.files['torrentfile'][0] - if (!req.body.targetUrl && !req.body.magnetUri && !file) { + if (!req.body.targetUrl && !req.body.magnetUri && !torrentFile) { cleanUpReqFiles(req) return res.status(400) diff --git a/server/models/account/user.ts b/server/models/account/user.ts index 1165285ea..1b1fc5ee8 100644 --- a/server/models/account/user.ts +++ b/server/models/account/user.ts @@ -295,7 +295,7 @@ export class UserModel extends Model { return json } - isAbleToUploadVideo (videoFile: Express.Multer.File) { + isAbleToUploadVideo (videoFile: { size: number }) { if (this.videoQuota === -1) return Promise.resolve(true) return UserModel.getOriginalVideoFileTotalFromUser(this) diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts index d6c02e5ac..b794d8324 100644 --- a/server/models/video/video-import.ts +++ b/server/models/video/video-import.ts @@ -15,34 +15,21 @@ import { } from 'sequelize-typescript' import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers' import { getSort, throwIfNotValid } from '../utils' -import { VideoModel } from './video' +import { ScopeNames as VideoModelScopeNames, VideoModel } from './video' import { isVideoImportStateValid, isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports' import { VideoImport, VideoImportState } from '../../../shared' -import { VideoChannelModel } from './video-channel' -import { AccountModel } from '../account/account' -import { TagModel } from './tag' import { isVideoMagnetUriValid } from '../../helpers/custom-validators/videos' +import { UserModel } from '../account/user' @DefaultScope({ include: [ { - model: () => VideoModel, - required: false, - include: [ - { - model: () => VideoChannelModel, - required: true, - include: [ - { - model: () => AccountModel, - required: true - } - ] - }, - { - model: () => TagModel - } - ] + model: () => UserModel.unscoped(), + required: true + }, + { + model: () => VideoModel.scope([ VideoModelScopeNames.WITH_ACCOUNT_DETAILS, VideoModelScopeNames.WITH_TAGS]), + required: false } ] }) @@ -53,6 +40,9 @@ import { isVideoMagnetUriValid } from '../../helpers/custom-validators/videos' { fields: [ 'videoId' ], unique: true + }, + { + fields: [ 'userId' ] } ] }) @@ -91,6 +81,18 @@ export class VideoImportModel extends Model { @Column(DataType.TEXT) error: string + @ForeignKey(() => UserModel) + @Column + userId: number + + @BelongsTo(() => UserModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + User: UserModel + @ForeignKey(() => VideoModel) @Column videoId: number @@ -116,41 +118,24 @@ export class VideoImportModel extends Model { return VideoImportModel.findById(id) } - static listUserVideoImportsForApi (accountId: number, start: number, count: number, sort: string) { + static listUserVideoImportsForApi (userId: number, start: number, count: number, sort: string) { const query = { distinct: true, - offset: start, - limit: count, - order: getSort(sort), include: [ { - model: VideoModel, - required: false, - include: [ - { - model: VideoChannelModel, - required: true, - include: [ - { - model: AccountModel, - required: true, - where: { - id: accountId - } - } - ] - }, - { - model: TagModel, - required: false - } - ] + model: UserModel.unscoped(), // FIXME: Without this, sequelize try to COUNT(DISTINCT(*)) which is an invalid SQL query + required: true } - ] + ], + offset: start, + limit: count, + order: getSort(sort), + where: { + userId + } } - return VideoImportModel.unscoped() - .findAndCountAll(query) + return VideoImportModel.findAndCountAll(query) .then(({ rows, count }) => { return { data: rows, diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index 2742e26de..b26dfa252 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts @@ -65,6 +65,9 @@ describe('Test config API validators', function () { videos: { http: { enabled: false + }, + torrent: { + enabled: false } } } diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index b65061a5d..f9805b6ea 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts @@ -45,6 +45,7 @@ function checkInitialConfig (data: CustomConfig) { expect(data.transcoding.resolutions['720p']).to.be.true expect(data.transcoding.resolutions['1080p']).to.be.true expect(data.import.videos.http.enabled).to.be.true + expect(data.import.videos.torrent.enabled).to.be.true } function checkUpdatedConfig (data: CustomConfig) { @@ -72,6 +73,7 @@ function checkUpdatedConfig (data: CustomConfig) { expect(data.transcoding.resolutions['720p']).to.be.false expect(data.transcoding.resolutions['1080p']).to.be.false expect(data.import.videos.http.enabled).to.be.false + expect(data.import.videos.torrent.enabled).to.be.false } describe('Test config', function () { @@ -167,6 +169,9 @@ describe('Test config', function () { videos: { http: { enabled: false + }, + torrent: { + enabled: false } } } diff --git a/server/tests/utils/server/config.ts b/server/tests/utils/server/config.ts index e21614282..d6ac3ef8a 100644 --- a/server/tests/utils/server/config.ts +++ b/server/tests/utils/server/config.ts @@ -97,6 +97,9 @@ function updateCustomSubConfig (url: string, token: string, newConfig: any) { videos: { http: { enabled: false + }, + torrent: { + enabled: false } } } diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts index 46320435d..d70c757b6 100644 --- a/shared/models/server/custom-config.model.ts +++ b/shared/models/server/custom-config.model.ts @@ -60,6 +60,9 @@ export interface CustomConfig { videos: { http: { enabled: boolean + }, + torrent: { + enabled: boolean } } } diff --git a/shared/models/server/server-config.model.ts b/shared/models/server/server-config.model.ts index 2cafedbbc..8cb087234 100644 --- a/shared/models/server/server-config.model.ts +++ b/shared/models/server/server-config.model.ts @@ -28,6 +28,9 @@ export interface ServerConfig { http: { enabled: boolean } + torrent: { + enabled: boolean + } } } -- 2.41.0