From 65fcc3119c334b75dd13bcfdebf186afdc580a8f Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 15 May 2017 22:22:03 +0200 Subject: First typescript iteration --- .gitignore | 1 + package.json | 16 + server.js | 154 ---- server.ts | 142 ++++ server/controllers/api/clients.js | 41 - server/controllers/api/clients.ts | 41 + server/controllers/api/config.js | 22 - server/controllers/api/config.ts | 22 + server/controllers/api/index.js | 35 - server/controllers/api/index.ts | 33 + server/controllers/api/pods.js | 109 --- server/controllers/api/pods.ts | 118 +++ server/controllers/api/remote/index.js | 18 - server/controllers/api/remote/index.ts | 18 + server/controllers/api/remote/pods.js | 42 - server/controllers/api/remote/pods.ts | 40 + server/controllers/api/remote/videos.js | 509 ------------ server/controllers/api/remote/videos.ts | 521 ++++++++++++ server/controllers/api/requests.js | 55 -- server/controllers/api/requests.ts | 57 ++ server/controllers/api/users.js | 169 ---- server/controllers/api/users.ts | 173 ++++ server/controllers/api/videos/abuse.js | 112 --- server/controllers/api/videos/abuse.ts | 117 +++ server/controllers/api/videos/blacklist.js | 43 - server/controllers/api/videos/blacklist.ts | 43 + server/controllers/api/videos/index.js | 404 ---------- server/controllers/api/videos/index.ts | 426 ++++++++++ server/controllers/api/videos/rate.js | 169 ---- server/controllers/api/videos/rate.ts | 181 +++++ server/controllers/client.js | 110 --- server/controllers/client.ts | 118 +++ server/controllers/index.js | 11 - server/controllers/index.ts | 3 + server/controllers/static.js | 45 -- server/controllers/static.ts | 49 ++ server/helpers/custom-validators/index.js | 19 - server/helpers/custom-validators/index.ts | 6 + server/helpers/custom-validators/misc.js | 18 - server/helpers/custom-validators/misc.ts | 14 + server/helpers/custom-validators/pods.js | 26 - server/helpers/custom-validators/pods.ts | 24 + server/helpers/custom-validators/remote/index.js | 11 - server/helpers/custom-validators/remote/index.ts | 1 + server/helpers/custom-validators/remote/videos.js | 118 --- server/helpers/custom-validators/remote/videos.ts | 138 ++++ server/helpers/custom-validators/users.js | 36 - server/helpers/custom-validators/users.ts | 34 + server/helpers/custom-validators/videos.js | 148 ---- server/helpers/custom-validators/videos.ts | 153 ++++ server/helpers/database-utils.js | 72 -- server/helpers/database-utils.ts | 69 ++ server/helpers/index.ts | 6 + server/helpers/logger.js | 49 -- server/helpers/logger.ts | 48 ++ server/helpers/peertube-crypto.js | 168 ---- server/helpers/peertube-crypto.ts | 171 ++++ server/helpers/requests.js | 68 -- server/helpers/requests.ts | 65 ++ server/helpers/utils.js | 58 -- server/helpers/utils.ts | 54 ++ server/initializers/checker.js | 88 --- server/initializers/checker.ts | 84 ++ server/initializers/constants.js | 343 -------- server/initializers/constants.ts | 343 ++++++++ server/initializers/database.js | 77 -- server/initializers/database.ts | 72 ++ server/initializers/index.ts | 6 + server/initializers/installer.js | 134 ---- server/initializers/installer.ts | 128 +++ server/initializers/migrations/0005-email-pod.js | 41 - server/initializers/migrations/0005-email-pod.ts | 44 ++ server/initializers/migrations/0010-email-user.js | 41 - server/initializers/migrations/0010-email-user.ts | 44 ++ server/initializers/migrations/0015-video-views.js | 19 - server/initializers/migrations/0015-video-views.ts | 22 + server/initializers/migrations/0020-video-likes.js | 19 - server/initializers/migrations/0020-video-likes.ts | 22 + .../initializers/migrations/0025-video-dislikes.js | 19 - .../initializers/migrations/0025-video-dislikes.ts | 22 + .../initializers/migrations/0030-video-category.js | 34 - .../initializers/migrations/0030-video-category.ts | 37 + .../initializers/migrations/0035-video-licence.js | 34 - .../initializers/migrations/0035-video-licence.ts | 37 + server/initializers/migrations/0040-video-nsfw.js | 34 - server/initializers/migrations/0040-video-nsfw.ts | 37 + .../migrations/0045-user-display-nsfw.js | 19 - .../migrations/0045-user-display-nsfw.ts | 22 + .../initializers/migrations/0050-video-language.js | 19 - .../initializers/migrations/0050-video-language.ts | 22 + server/initializers/migrator.js | 139 ---- server/initializers/migrator.ts | 134 ++++ server/lib/friends.js | 405 ---------- server/lib/friends.ts | 410 ++++++++++ server/lib/index.ts | 4 + server/lib/jobs/handlers/index.js | 7 - server/lib/jobs/handlers/index.ts | 9 + server/lib/jobs/handlers/video-transcoder.js | 43 - server/lib/jobs/handlers/video-transcoder.ts | 37 + server/lib/jobs/index.ts | 1 + server/lib/jobs/job-scheduler.js | 129 --- server/lib/jobs/job-scheduler.ts | 137 ++++ server/lib/oauth-model.js | 97 --- server/lib/oauth-model.ts | 95 +++ server/lib/request/base-request-scheduler.js | 136 ---- server/lib/request/base-request-scheduler.ts | 154 ++++ server/lib/request/index.ts | 3 + server/lib/request/request-scheduler.js | 97 --- server/lib/request/request-scheduler.ts | 104 +++ .../lib/request/request-video-event-scheduler.js | 108 --- .../lib/request/request-video-event-scheduler.ts | 116 +++ server/lib/request/request-video-qadu-scheduler.js | 117 --- server/lib/request/request-video-qadu-scheduler.ts | 126 +++ server/middlewares/admin.js | 21 - server/middlewares/admin.ts | 17 + server/middlewares/index.js | 25 - server/middlewares/index.ts | 8 + server/middlewares/oauth.js | 38 - server/middlewares/oauth.ts | 34 + server/middlewares/pagination.js | 20 - server/middlewares/pagination.ts | 17 + server/middlewares/pods.js | 59 -- server/middlewares/pods.ts | 57 ++ server/middlewares/search.js | 15 - server/middlewares/search.ts | 11 + server/middlewares/secure.js | 52 -- server/middlewares/secure.ts | 48 ++ server/middlewares/sort.js | 29 - server/middlewares/sort.ts | 25 + server/middlewares/validators/index.js | 21 - server/middlewares/validators/index.ts | 6 + server/middlewares/validators/pagination.js | 21 - server/middlewares/validators/pagination.ts | 17 + server/middlewares/validators/pods.js | 67 -- server/middlewares/validators/pods.ts | 63 ++ server/middlewares/validators/remote/index.js | 13 - server/middlewares/validators/remote/index.ts | 2 + server/middlewares/validators/remote/signature.js | 21 - server/middlewares/validators/remote/signature.ts | 17 + server/middlewares/validators/remote/videos.js | 37 - server/middlewares/validators/remote/videos.ts | 34 + server/middlewares/validators/sort.js | 48 -- server/middlewares/validators/sort.ts | 44 ++ server/middlewares/validators/users.js | 88 --- server/middlewares/validators/users.ts | 84 ++ server/middlewares/validators/utils.js | 25 - server/middlewares/validators/utils.ts | 21 + server/middlewares/validators/videos.js | 204 ----- server/middlewares/validators/videos.ts | 199 +++++ server/models/application.js | 52 -- server/models/application.ts | 50 ++ server/models/author.js | 92 --- server/models/author.ts | 90 +++ server/models/job.js | 54 -- server/models/job.ts | 52 ++ server/models/oauth-client.js | 62 -- server/models/oauth-client.ts | 60 ++ server/models/oauth-token.js | 144 ---- server/models/oauth-token.ts | 142 ++++ server/models/pod.js | 273 ------- server/models/pod.ts | 269 +++++++ server/models/request-to-pod.js | 42 - server/models/request-to-pod.ts | 38 + server/models/request-video-event.js | 172 ---- server/models/request-video-event.ts | 170 ++++ server/models/request-video-qadu.js | 151 ---- server/models/request-video-qadu.ts | 149 ++++ server/models/request.js | 137 ---- server/models/request.ts | 135 ++++ server/models/tag.js | 76 -- server/models/tag.ts | 74 ++ server/models/user-video-rate.js | 77 -- server/models/user-video-rate.ts | 74 ++ server/models/user.js | 194 ----- server/models/user.ts | 197 +++++ server/models/utils.js | 25 - server/models/utils.ts | 21 + server/models/video-abuse.js | 114 --- server/models/video-abuse.ts | 112 +++ server/models/video-blacklist.js | 89 --- server/models/video-blacklist.ts | 87 ++ server/models/video-tag.js | 18 - server/models/video-tag.ts | 14 + server/models/video.js | 858 -------------------- server/models/video.ts | 873 +++++++++++++++++++++ tsconfig.json | 19 + tslint.json | 3 + yarn.lock | 166 +++- 188 files changed, 8661 insertions(+), 8484 deletions(-) delete mode 100644 server.js create mode 100644 server.ts delete mode 100644 server/controllers/api/clients.js create mode 100644 server/controllers/api/clients.ts delete mode 100644 server/controllers/api/config.js create mode 100644 server/controllers/api/config.ts delete mode 100644 server/controllers/api/index.js create mode 100644 server/controllers/api/index.ts delete mode 100644 server/controllers/api/pods.js create mode 100644 server/controllers/api/pods.ts delete mode 100644 server/controllers/api/remote/index.js create mode 100644 server/controllers/api/remote/index.ts delete mode 100644 server/controllers/api/remote/pods.js create mode 100644 server/controllers/api/remote/pods.ts delete mode 100644 server/controllers/api/remote/videos.js create mode 100644 server/controllers/api/remote/videos.ts delete mode 100644 server/controllers/api/requests.js create mode 100644 server/controllers/api/requests.ts delete mode 100644 server/controllers/api/users.js create mode 100644 server/controllers/api/users.ts delete mode 100644 server/controllers/api/videos/abuse.js create mode 100644 server/controllers/api/videos/abuse.ts delete mode 100644 server/controllers/api/videos/blacklist.js create mode 100644 server/controllers/api/videos/blacklist.ts delete mode 100644 server/controllers/api/videos/index.js create mode 100644 server/controllers/api/videos/index.ts delete mode 100644 server/controllers/api/videos/rate.js create mode 100644 server/controllers/api/videos/rate.ts delete mode 100644 server/controllers/client.js create mode 100644 server/controllers/client.ts delete mode 100644 server/controllers/index.js create mode 100644 server/controllers/index.ts delete mode 100644 server/controllers/static.js create mode 100644 server/controllers/static.ts delete mode 100644 server/helpers/custom-validators/index.js create mode 100644 server/helpers/custom-validators/index.ts delete mode 100644 server/helpers/custom-validators/misc.js create mode 100644 server/helpers/custom-validators/misc.ts delete mode 100644 server/helpers/custom-validators/pods.js create mode 100644 server/helpers/custom-validators/pods.ts delete mode 100644 server/helpers/custom-validators/remote/index.js create mode 100644 server/helpers/custom-validators/remote/index.ts delete mode 100644 server/helpers/custom-validators/remote/videos.js create mode 100644 server/helpers/custom-validators/remote/videos.ts delete mode 100644 server/helpers/custom-validators/users.js create mode 100644 server/helpers/custom-validators/users.ts delete mode 100644 server/helpers/custom-validators/videos.js create mode 100644 server/helpers/custom-validators/videos.ts delete mode 100644 server/helpers/database-utils.js create mode 100644 server/helpers/database-utils.ts create mode 100644 server/helpers/index.ts delete mode 100644 server/helpers/logger.js create mode 100644 server/helpers/logger.ts delete mode 100644 server/helpers/peertube-crypto.js create mode 100644 server/helpers/peertube-crypto.ts delete mode 100644 server/helpers/requests.js create mode 100644 server/helpers/requests.ts delete mode 100644 server/helpers/utils.js create mode 100644 server/helpers/utils.ts delete mode 100644 server/initializers/checker.js create mode 100644 server/initializers/checker.ts delete mode 100644 server/initializers/constants.js create mode 100644 server/initializers/constants.ts delete mode 100644 server/initializers/database.js create mode 100644 server/initializers/database.ts create mode 100644 server/initializers/index.ts delete mode 100644 server/initializers/installer.js create mode 100644 server/initializers/installer.ts delete mode 100644 server/initializers/migrations/0005-email-pod.js create mode 100644 server/initializers/migrations/0005-email-pod.ts delete mode 100644 server/initializers/migrations/0010-email-user.js create mode 100644 server/initializers/migrations/0010-email-user.ts delete mode 100644 server/initializers/migrations/0015-video-views.js create mode 100644 server/initializers/migrations/0015-video-views.ts delete mode 100644 server/initializers/migrations/0020-video-likes.js create mode 100644 server/initializers/migrations/0020-video-likes.ts delete mode 100644 server/initializers/migrations/0025-video-dislikes.js create mode 100644 server/initializers/migrations/0025-video-dislikes.ts delete mode 100644 server/initializers/migrations/0030-video-category.js create mode 100644 server/initializers/migrations/0030-video-category.ts delete mode 100644 server/initializers/migrations/0035-video-licence.js create mode 100644 server/initializers/migrations/0035-video-licence.ts delete mode 100644 server/initializers/migrations/0040-video-nsfw.js create mode 100644 server/initializers/migrations/0040-video-nsfw.ts delete mode 100644 server/initializers/migrations/0045-user-display-nsfw.js create mode 100644 server/initializers/migrations/0045-user-display-nsfw.ts delete mode 100644 server/initializers/migrations/0050-video-language.js create mode 100644 server/initializers/migrations/0050-video-language.ts delete mode 100644 server/initializers/migrator.js create mode 100644 server/initializers/migrator.ts delete mode 100644 server/lib/friends.js create mode 100644 server/lib/friends.ts create mode 100644 server/lib/index.ts delete mode 100644 server/lib/jobs/handlers/index.js create mode 100644 server/lib/jobs/handlers/index.ts delete mode 100644 server/lib/jobs/handlers/video-transcoder.js create mode 100644 server/lib/jobs/handlers/video-transcoder.ts create mode 100644 server/lib/jobs/index.ts delete mode 100644 server/lib/jobs/job-scheduler.js create mode 100644 server/lib/jobs/job-scheduler.ts delete mode 100644 server/lib/oauth-model.js create mode 100644 server/lib/oauth-model.ts delete mode 100644 server/lib/request/base-request-scheduler.js create mode 100644 server/lib/request/base-request-scheduler.ts create mode 100644 server/lib/request/index.ts delete mode 100644 server/lib/request/request-scheduler.js create mode 100644 server/lib/request/request-scheduler.ts delete mode 100644 server/lib/request/request-video-event-scheduler.js create mode 100644 server/lib/request/request-video-event-scheduler.ts delete mode 100644 server/lib/request/request-video-qadu-scheduler.js create mode 100644 server/lib/request/request-video-qadu-scheduler.ts delete mode 100644 server/middlewares/admin.js create mode 100644 server/middlewares/admin.ts delete mode 100644 server/middlewares/index.js create mode 100644 server/middlewares/index.ts delete mode 100644 server/middlewares/oauth.js create mode 100644 server/middlewares/oauth.ts delete mode 100644 server/middlewares/pagination.js create mode 100644 server/middlewares/pagination.ts delete mode 100644 server/middlewares/pods.js create mode 100644 server/middlewares/pods.ts delete mode 100644 server/middlewares/search.js create mode 100644 server/middlewares/search.ts delete mode 100644 server/middlewares/secure.js create mode 100644 server/middlewares/secure.ts delete mode 100644 server/middlewares/sort.js create mode 100644 server/middlewares/sort.ts delete mode 100644 server/middlewares/validators/index.js create mode 100644 server/middlewares/validators/index.ts delete mode 100644 server/middlewares/validators/pagination.js create mode 100644 server/middlewares/validators/pagination.ts delete mode 100644 server/middlewares/validators/pods.js create mode 100644 server/middlewares/validators/pods.ts delete mode 100644 server/middlewares/validators/remote/index.js create mode 100644 server/middlewares/validators/remote/index.ts delete mode 100644 server/middlewares/validators/remote/signature.js create mode 100644 server/middlewares/validators/remote/signature.ts delete mode 100644 server/middlewares/validators/remote/videos.js create mode 100644 server/middlewares/validators/remote/videos.ts delete mode 100644 server/middlewares/validators/sort.js create mode 100644 server/middlewares/validators/sort.ts delete mode 100644 server/middlewares/validators/users.js create mode 100644 server/middlewares/validators/users.ts delete mode 100644 server/middlewares/validators/utils.js create mode 100644 server/middlewares/validators/utils.ts delete mode 100644 server/middlewares/validators/videos.js create mode 100644 server/middlewares/validators/videos.ts delete mode 100644 server/models/application.js create mode 100644 server/models/application.ts delete mode 100644 server/models/author.js create mode 100644 server/models/author.ts delete mode 100644 server/models/job.js create mode 100644 server/models/job.ts delete mode 100644 server/models/oauth-client.js create mode 100644 server/models/oauth-client.ts delete mode 100644 server/models/oauth-token.js create mode 100644 server/models/oauth-token.ts delete mode 100644 server/models/pod.js create mode 100644 server/models/pod.ts delete mode 100644 server/models/request-to-pod.js create mode 100644 server/models/request-to-pod.ts delete mode 100644 server/models/request-video-event.js create mode 100644 server/models/request-video-event.ts delete mode 100644 server/models/request-video-qadu.js create mode 100644 server/models/request-video-qadu.ts delete mode 100644 server/models/request.js create mode 100644 server/models/request.ts delete mode 100644 server/models/tag.js create mode 100644 server/models/tag.ts delete mode 100644 server/models/user-video-rate.js create mode 100644 server/models/user-video-rate.ts delete mode 100644 server/models/user.js create mode 100644 server/models/user.ts delete mode 100644 server/models/utils.js create mode 100644 server/models/utils.ts delete mode 100644 server/models/video-abuse.js create mode 100644 server/models/video-abuse.ts delete mode 100644 server/models/video-blacklist.js create mode 100644 server/models/video-blacklist.ts delete mode 100644 server/models/video-tag.js create mode 100644 server/models/video-tag.ts delete mode 100644 server/models/video.js create mode 100644 server/models/video.ts create mode 100644 tsconfig.json create mode 100644 tslint.json diff --git a/.gitignore b/.gitignore index 28dec58f3..6caee2e4c 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ /ffmpeg/ /*.sublime-project /*.sublime-workspace +/dist diff --git a/package.json b/package.json index 9c63c67b4..00d0bb5ee 100644 --- a/package.json +++ b/package.json @@ -70,15 +70,31 @@ "safe-buffer": "^5.0.1", "scripty": "^1.5.0", "sequelize": "^3.27.0", + "typescript": "~2.2.0", "winston": "^2.1.1", "ws": "^2.0.0" }, "devDependencies": { + "@types/async": "^2.0.40", + "@types/bcrypt": "^1.0.0", + "@types/body-parser": "^1.16.3", + "@types/config": "^0.0.32", + "@types/express": "^4.0.35", + "@types/lodash": "^4.14.64", + "@types/mkdirp": "^0.3.29", + "@types/morgan": "^1.7.32", + "@types/node": "^7.0.18", + "@types/request": "^0.0.43", + "@types/sequelize": "3", + "@types/winston": "^2.3.2", + "@types/ws": "^0.0.41", "chai": "^3.3.0", "commander": "^2.9.0", "mocha": "^3.0.1", "standard": "^10.0.0", "supertest": "^3.0.0", + "tslint": "^5.2.0", + "tslint-config-standard": "^5.0.2", "webtorrent": "^0.98.0" }, "standard": { diff --git a/server.js b/server.js deleted file mode 100644 index b2487b767..000000000 --- a/server.js +++ /dev/null @@ -1,154 +0,0 @@ -'use strict' - -// ----------- Node modules ----------- -const bodyParser = require('body-parser') -const express = require('express') -const expressValidator = require('express-validator') -const http = require('http') -const morgan = require('morgan') -const path = require('path') -const TrackerServer = require('bittorrent-tracker').Server -const WebSocketServer = require('ws').Server - -process.title = 'peertube' - -// Create our main app -const app = express() - -// ----------- Database ----------- -const constants = require('./server/initializers/constants') -const logger = require('./server/helpers/logger') -// Initialize database and models -const db = require('./server/initializers/database') -db.init(onDatabaseInitDone) - -// ----------- Checker ----------- -const checker = require('./server/initializers/checker') - -const missed = checker.checkMissedConfig() -if (missed.length !== 0) { - throw new Error('Miss some configurations keys : ' + missed) -} -checker.checkFFmpeg(function (err) { - if (err) { - throw err - } -}) - -const errorMessage = checker.checkConfig() -if (errorMessage !== null) { - throw new Error(errorMessage) -} - -// ----------- PeerTube modules ----------- -const customValidators = require('./server/helpers/custom-validators') -const friends = require('./server/lib/friends') -const installer = require('./server/initializers/installer') -const migrator = require('./server/initializers/migrator') -const jobScheduler = require('./server/lib/jobs/job-scheduler') -const routes = require('./server/controllers') - -// ----------- Command line ----------- - -// ----------- App ----------- - -// For the logger -app.use(morgan('combined', { stream: logger.stream })) -// For body requests -app.use(bodyParser.json({ limit: '500kb' })) -app.use(bodyParser.urlencoded({ extended: false })) -// Validate some params for the API -app.use(expressValidator({ - customValidators: Object.assign( - {}, - customValidators.misc, - customValidators.pods, - customValidators.users, - customValidators.videos, - customValidators.remote.videos - ) -})) - -// ----------- Views, routes and static files ----------- - -// API -const apiRoute = '/api/' + constants.API_VERSION -app.use(apiRoute, routes.api) - -// Client files -app.use('/', routes.client) - -// Static files -app.use('/', routes.static) - -// Always serve index client page (the client is a single page application, let it handle routing) -app.use('/*', function (req, res, next) { - res.sendFile(path.join(__dirname, './client/dist/index.html')) -}) - -// ----------- Tracker ----------- - -const trackerServer = new TrackerServer({ - http: false, - udp: false, - ws: false, - dht: false -}) - -trackerServer.on('error', function (err) { - logger.error(err) -}) - -trackerServer.on('warning', function (err) { - logger.error(err) -}) - -const server = http.createServer(app) -const wss = new WebSocketServer({server: server, path: '/tracker/socket'}) -wss.on('connection', function (ws) { - trackerServer.onWebSocketConnection(ws) -}) - -// ----------- Errors ----------- - -// Catch 404 and forward to error handler -app.use(function (req, res, next) { - const err = new Error('Not Found') - err.status = 404 - next(err) -}) - -app.use(function (err, req, res, next) { - logger.error(err) - res.sendStatus(err.status || 500) -}) - -// ----------- Run ----------- - -function onDatabaseInitDone () { - const port = constants.CONFIG.LISTEN.PORT - // Run the migration scripts if needed - migrator.migrate(function (err) { - if (err) throw err - - installer.installApplication(function (err) { - if (err) throw err - - // ----------- Make the server listening ----------- - server.listen(port, function () { - // Activate the communication with friends - friends.activate() - - // Activate job scheduler - jobScheduler.activate() - - logger.info('Server listening on port %d', port) - logger.info('Webserver: %s', constants.CONFIG.WEBSERVER.URL) - - app.emit('ready') - }) - }) - }) -} - -module.exports = app diff --git a/server.ts b/server.ts new file mode 100644 index 000000000..119c0c61d --- /dev/null +++ b/server.ts @@ -0,0 +1,142 @@ +// ----------- Node modules ----------- +import bodyParser = require('body-parser') +import express = require('express') +const expressValidator = require('express-validator') +import http = require('http') +import morgan = require('morgan') +import path = require('path') +import bittorrentTracker = require('bittorrent-tracker') +import { Server as WebSocketServer } from 'ws' + +const TrackerServer = bittorrentTracker.Server + +process.title = 'peertube' + +// Create our main app +const app = express() + +// ----------- Database ----------- +// Do not use barels because we don't want to load all modules here (we need to initialize database first) +import { logger } from './server/helpers/logger' +import { API_VERSION, CONFIG } from './server/initializers/constants' +// Initialize database and models +const db = require('./server/initializers/database') +db.init(onDatabaseInitDone) + +// ----------- Checker ----------- +import { checkMissedConfig, checkFFmpeg, checkConfig } from './server/initializers/checker' + +const missed = checkMissedConfig() +if (missed.length !== 0) { + throw new Error('Miss some configurations keys : ' + missed) +} +checkFFmpeg(function (err) { + if (err) { + throw err + } +}) + +const errorMessage = checkConfig() +if (errorMessage !== null) { + throw new Error(errorMessage) +} + +// ----------- PeerTube modules ----------- +import { migrate, installApplication } from './server/initializers' +import { JobScheduler, activateSchedulers } from './server/lib' +import * as customValidators from './server/helpers/custom-validators' +import { apiRouter, clientsRouter, staticRouter } from './server/controllers' + +// ----------- Command line ----------- + +// ----------- App ----------- + +// For the logger +// app.use(morgan('combined', { stream: logger.stream })) +// For body requests +app.use(bodyParser.json({ limit: '500kb' })) +app.use(bodyParser.urlencoded({ extended: false })) +// Validate some params for the API +app.use(expressValidator({ + customValidators: customValidators +})) + +// ----------- Views, routes and static files ----------- + +// API +const apiRoute = '/api/' + API_VERSION +app.use(apiRoute, apiRouter) + +// Client files +app.use('/', clientsRouter) + +// Static files +app.use('/', staticRouter) + +// Always serve index client page (the client is a single page application, let it handle routing) +app.use('/*', function (req, res, next) { + res.sendFile(path.join(__dirname, './client/dist/index.html')) +}) + +// ----------- Tracker ----------- + +const trackerServer = new TrackerServer({ + http: false, + udp: false, + ws: false, + dht: false +}) + +trackerServer.on('error', function (err) { + logger.error(err) +}) + +trackerServer.on('warning', function (err) { + logger.error(err) +}) + +const server = http.createServer(app) +const wss = new WebSocketServer({ server: server, path: '/tracker/socket' }) +wss.on('connection', function (ws) { + trackerServer.onWebSocketConnection(ws) +}) + +// ----------- Errors ----------- + +// Catch 404 and forward to error handler +app.use(function (req, res, next) { + const err = new Error('Not Found') + err['status'] = 404 + next(err) +}) + +app.use(function (err, req, res, next) { + logger.error(err) + res.sendStatus(err.status || 500) +}) + +// ----------- Run ----------- + +function onDatabaseInitDone () { + const port = CONFIG.LISTEN.PORT + // Run the migration scripts if needed + migrate(function (err) { + if (err) throw err + + installApplication(function (err) { + if (err) throw err + + // ----------- Make the server listening ----------- + server.listen(port, function () { + // Activate the communication with friends + activateSchedulers() + + // Activate job scheduler + JobScheduler.Instance.activate() + + logger.info('Server listening on port %d', port) + logger.info('Webserver: %s', CONFIG.WEBSERVER.URL) + }) + }) + }) +} diff --git a/server/controllers/api/clients.js b/server/controllers/api/clients.js deleted file mode 100644 index cf83cb835..000000000 --- a/server/controllers/api/clients.js +++ /dev/null @@ -1,41 +0,0 @@ -'use strict' - -const express = require('express') - -const constants = require('../../initializers/constants') -const db = require('../../initializers/database') -const logger = require('../../helpers/logger') - -const router = express.Router() - -router.get('/local', getLocalClient) - -// Get the client credentials for the PeerTube front end -function getLocalClient (req, res, next) { - const serverHostname = constants.CONFIG.WEBSERVER.HOSTNAME - const serverPort = constants.CONFIG.WEBSERVER.PORT - let headerHostShouldBe = serverHostname - if (serverPort !== 80 && serverPort !== 443) { - headerHostShouldBe += ':' + serverPort - } - - // Don't make this check if this is a test instance - if (process.env.NODE_ENV !== 'test' && req.get('host') !== headerHostShouldBe) { - logger.info('Getting client tokens for host %s is forbidden (expected %s).', req.get('host'), headerHostShouldBe) - return res.type('json').status(403).end() - } - - db.OAuthClient.loadFirstClient(function (err, client) { - if (err) return next(err) - if (!client) return next(new Error('No client available.')) - - res.json({ - client_id: client.clientId, - client_secret: client.clientSecret - }) - }) -} - -// --------------------------------------------------------------------------- - -module.exports = router diff --git a/server/controllers/api/clients.ts b/server/controllers/api/clients.ts new file mode 100644 index 000000000..902f62995 --- /dev/null +++ b/server/controllers/api/clients.ts @@ -0,0 +1,41 @@ +import express = require('express') + +import { CONFIG } from '../../initializers'; +import { logger } from '../../helpers' +const db = require('../../initializers/database') + +const clientsRouter = express.Router() + +clientsRouter.get('/local', getLocalClient) + +// Get the client credentials for the PeerTube front end +function getLocalClient (req, res, next) { + const serverHostname = CONFIG.WEBSERVER.HOSTNAME + const serverPort = CONFIG.WEBSERVER.PORT + let headerHostShouldBe = serverHostname + if (serverPort !== 80 && serverPort !== 443) { + headerHostShouldBe += ':' + serverPort + } + + // Don't make this check if this is a test instance + if (process.env.NODE_ENV !== 'test' && req.get('host') !== headerHostShouldBe) { + logger.info('Getting client tokens for host %s is forbidden (expected %s).', req.get('host'), headerHostShouldBe) + return res.type('json').status(403).end() + } + + db.OAuthClient.loadFirstClient(function (err, client) { + if (err) return next(err) + if (!client) return next(new Error('No client available.')) + + res.json({ + client_id: client.clientId, + client_secret: client.clientSecret + }) + }) +} + +// --------------------------------------------------------------------------- + +export { + clientsRouter +} diff --git a/server/controllers/api/config.js b/server/controllers/api/config.js deleted file mode 100644 index 8154b6ad0..000000000 --- a/server/controllers/api/config.js +++ /dev/null @@ -1,22 +0,0 @@ -'use strict' - -const express = require('express') - -const constants = require('../../initializers/constants') - -const router = express.Router() - -router.get('/', getConfig) - -// Get the client credentials for the PeerTube front end -function getConfig (req, res, next) { - res.json({ - signup: { - enabled: constants.CONFIG.SIGNUP.ENABLED - } - }) -} - -// --------------------------------------------------------------------------- - -module.exports = router diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts new file mode 100644 index 000000000..8f3fa2473 --- /dev/null +++ b/server/controllers/api/config.ts @@ -0,0 +1,22 @@ +import express = require('express') + +import { CONFIG } from '../../initializers'; + +const configRouter = express.Router() + +configRouter.get('/', getConfig) + +// Get the client credentials for the PeerTube front end +function getConfig (req, res, next) { + res.json({ + signup: { + enabled: CONFIG.SIGNUP.ENABLED + } + }) +} + +// --------------------------------------------------------------------------- + +export { + configRouter +} diff --git a/server/controllers/api/index.js b/server/controllers/api/index.js deleted file mode 100644 index 6edc089f4..000000000 --- a/server/controllers/api/index.js +++ /dev/null @@ -1,35 +0,0 @@ -'use strict' - -const express = require('express') - -const utils = require('../../helpers/utils') - -const router = express.Router() - -const clientsController = require('./clients') -const configController = require('./config') -const podsController = require('./pods') -const remoteController = require('./remote') -const requestsController = require('./requests') -const usersController = require('./users') -const videosController = require('./videos') - -router.use('/clients', clientsController) -router.use('/config', configController) -router.use('/pods', podsController) -router.use('/remote', remoteController) -router.use('/requests', requestsController) -router.use('/users', usersController) -router.use('/videos', videosController) -router.use('/ping', pong) -router.use('/*', utils.badRequest) - -// --------------------------------------------------------------------------- - -module.exports = router - -// --------------------------------------------------------------------------- - -function pong (req, res, next) { - return res.send('pong').status(200).end() -} diff --git a/server/controllers/api/index.ts b/server/controllers/api/index.ts new file mode 100644 index 000000000..18bef2d3d --- /dev/null +++ b/server/controllers/api/index.ts @@ -0,0 +1,33 @@ +import express = require('express') + +import { badRequest } from '../../helpers' + +import { clientsRouter } from './clients' +import { configRouter } from './config' +import { podsRouter } from './pods' +import { remoteRouter } from './remote' +import { requestsRouter } from './requests' +import { usersRouter } from './users' +import { videosRouter } from './videos' + +const apiRouter = express.Router() + +apiRouter.use('/clients', clientsRouter) +apiRouter.use('/config', configRouter) +apiRouter.use('/pods', podsRouter) +apiRouter.use('/remote', remoteRouter) +apiRouter.use('/requests', requestsRouter) +apiRouter.use('/users', usersRouter) +apiRouter.use('/videos', videosRouter) +apiRouter.use('/ping', pong) +apiRouter.use('/*', badRequest) + +// --------------------------------------------------------------------------- + +export { apiRouter } + +// --------------------------------------------------------------------------- + +function pong (req, res, next) { + return res.send('pong').status(200).end() +} diff --git a/server/controllers/api/pods.js b/server/controllers/api/pods.js deleted file mode 100644 index ab5763cf6..000000000 --- a/server/controllers/api/pods.js +++ /dev/null @@ -1,109 +0,0 @@ -'use strict' - -const express = require('express') -const waterfall = require('async/waterfall') - -const db = require('../../initializers/database') -const constants = require('../../initializers/constants') -const logger = require('../../helpers/logger') -const peertubeCrypto = require('../../helpers/peertube-crypto') -const utils = require('../../helpers/utils') -const friends = require('../../lib/friends') -const middlewares = require('../../middlewares') -const admin = middlewares.admin -const oAuth = middlewares.oauth -const podsMiddleware = middlewares.pods -const validators = middlewares.validators.pods - -const router = express.Router() - -router.get('/', listPods) -router.post('/', - podsMiddleware.setBodyHostPort, // We need to modify the host before running the validator! - validators.podsAdd, - addPods -) -router.post('/makefriends', - oAuth.authenticate, - admin.ensureIsAdmin, - validators.makeFriends, - podsMiddleware.setBodyHostsPort, - makeFriends -) -router.get('/quitfriends', - oAuth.authenticate, - admin.ensureIsAdmin, - quitFriends -) - -// --------------------------------------------------------------------------- - -module.exports = router - -// --------------------------------------------------------------------------- - -function addPods (req, res, next) { - const informations = req.body - - waterfall([ - function addPod (callback) { - const pod = db.Pod.build(informations) - pod.save().asCallback(function (err, podCreated) { - // Be sure about the number of parameters for the callback - return callback(err, podCreated) - }) - }, - - function sendMyVideos (podCreated, callback) { - friends.sendOwnedVideosToPod(podCreated.id) - - callback(null) - }, - - function fetchMyCertificate (callback) { - peertubeCrypto.getMyPublicCert(function (err, cert) { - if (err) { - logger.error('Cannot read cert file.') - return callback(err) - } - - return callback(null, cert) - }) - } - ], function (err, cert) { - if (err) return next(err) - - return res.json({ cert: cert, email: constants.CONFIG.ADMIN.EMAIL }) - }) -} - -function listPods (req, res, next) { - db.Pod.list(function (err, podsList) { - if (err) return next(err) - - res.json(utils.getFormatedObjects(podsList, podsList.length)) - }) -} - -function makeFriends (req, res, next) { - const hosts = req.body.hosts - - friends.makeFriends(hosts, function (err) { - if (err) { - logger.error('Could not make friends.', { error: err }) - return - } - - logger.info('Made friends!') - }) - - res.type('json').status(204).end() -} - -function quitFriends (req, res, next) { - friends.quitFriends(function (err) { - if (err) return next(err) - - res.type('json').status(204).end() - }) -} diff --git a/server/controllers/api/pods.ts b/server/controllers/api/pods.ts new file mode 100644 index 000000000..06dfd8295 --- /dev/null +++ b/server/controllers/api/pods.ts @@ -0,0 +1,118 @@ +import express = require('express') +import { waterfall } from 'async' + +const db = require('../../initializers/database') +import { CONFIG } from '../../initializers' +import { + logger, + getMyPublicCert, + getFormatedObjects +} from '../../helpers' +import { + sendOwnedVideosToPod, + makeFriends, + quitFriends +} from '../../lib' +import { + podsAddValidator, + authenticate, + ensureIsAdmin, + makeFriendsValidator, + setBodyHostPort, + setBodyHostsPort +} from '../../middlewares' + +const podsRouter = express.Router() + +podsRouter.get('/', listPods) +podsRouter.post('/', + setBodyHostPort, // We need to modify the host before running the validator! + podsAddValidator, + addPods +) +podsRouter.post('/makefriends', + authenticate, + ensureIsAdmin, + makeFriendsValidator, + setBodyHostsPort, + makeFriends +) +podsRouter.get('/quitfriends', + authenticate, + ensureIsAdmin, + quitFriends +) + +// --------------------------------------------------------------------------- + +export { + podsRouter +} + +// --------------------------------------------------------------------------- + +function addPods (req, res, next) { + const informations = req.body + + waterfall([ + function addPod (callback) { + const pod = db.Pod.build(informations) + pod.save().asCallback(function (err, podCreated) { + // Be sure about the number of parameters for the callback + return callback(err, podCreated) + }) + }, + + function sendMyVideos (podCreated, callback) { + sendOwnedVideosToPod(podCreated.id) + + callback(null) + }, + + function fetchMyCertificate (callback) { + getMyPublicCert(function (err, cert) { + if (err) { + logger.error('Cannot read cert file.') + return callback(err) + } + + return callback(null, cert) + }) + } + ], function (err, cert) { + if (err) return next(err) + + return res.json({ cert: cert, email: CONFIG.ADMIN.EMAIL }) + }) +} + +function listPods (req, res, next) { + db.Pod.list(function (err, podsList) { + if (err) return next(err) + + res.json(getFormatedObjects(podsList, podsList.length)) + }) +} + +function makeFriendsController (req, res, next) { + const hosts = req.body.hosts + + makeFriends(hosts, function (err) { + if (err) { + logger.error('Could not make friends.', { error: err }) + return + } + + logger.info('Made friends!') + }) + + res.type('json').status(204).end() +} + +function quitFriendsController (req, res, next) { + quitFriends(function (err) { + if (err) return next(err) + + res.type('json').status(204).end() + }) +} diff --git a/server/controllers/api/remote/index.js b/server/controllers/api/remote/index.js deleted file mode 100644 index 6106850ab..000000000 --- a/server/controllers/api/remote/index.js +++ /dev/null @@ -1,18 +0,0 @@ -'use strict' - -const express = require('express') - -const utils = require('../../../helpers/utils') - -const router = express.Router() - -const podsRemoteController = require('./pods') -const videosRemoteController = require('./videos') - -router.use('/pods', podsRemoteController) -router.use('/videos', videosRemoteController) -router.use('/*', utils.badRequest) - -// --------------------------------------------------------------------------- - -module.exports = router diff --git a/server/controllers/api/remote/index.ts b/server/controllers/api/remote/index.ts new file mode 100644 index 000000000..b11439204 --- /dev/null +++ b/server/controllers/api/remote/index.ts @@ -0,0 +1,18 @@ +import express = require('express') + +import { badRequest } from '../../../helpers' + +import { remotePodsRouter } from './pods' +import { remoteVideosRouter } from './videos' + +const remoteRouter = express.Router() + +remoteRouter.use('/pods', remotePodsRouter) +remoteRouter.use('/videos', remoteVideosRouter) +remoteRouter.use('/*', badRequest) + +// --------------------------------------------------------------------------- + +export { + remoteRouter +} diff --git a/server/controllers/api/remote/pods.js b/server/controllers/api/remote/pods.js deleted file mode 100644 index 0343bc62e..000000000 --- a/server/controllers/api/remote/pods.js +++ /dev/null @@ -1,42 +0,0 @@ -'use strict' - -const express = require('express') -const waterfall = require('async/waterfall') - -const db = require('../../../initializers/database') -const middlewares = require('../../../middlewares') -const checkSignature = middlewares.secure.checkSignature -const signatureValidator = middlewares.validators.remote.signature - -const router = express.Router() - -// Post because this is a secured request -router.post('/remove', - signatureValidator.signature, - checkSignature, - removePods -) - -// --------------------------------------------------------------------------- - -module.exports = router - -// --------------------------------------------------------------------------- - -function removePods (req, res, next) { - const host = req.body.signature.host - - waterfall([ - function loadPod (callback) { - db.Pod.loadByHost(host, callback) - }, - - function deletePod (pod, callback) { - pod.destroy().asCallback(callback) - } - ], function (err) { - if (err) return next(err) - - return res.type('json').status(204).end() - }) -} diff --git a/server/controllers/api/remote/pods.ts b/server/controllers/api/remote/pods.ts new file mode 100644 index 000000000..85ef7bb42 --- /dev/null +++ b/server/controllers/api/remote/pods.ts @@ -0,0 +1,40 @@ +import express = require('express') +import { waterfall } from 'async/waterfall' + +const db = require('../../../initializers/database') +import { checkSignature, signatureValidator } from '../../../middlewares' + +const remotePodsRouter = express.Router() + +// Post because this is a secured request +remotePodsRouter.post('/remove', + signatureValidator, + checkSignature, + removePods +) + +// --------------------------------------------------------------------------- + +export { + remotePodsRouter +} + +// --------------------------------------------------------------------------- + +function removePods (req, res, next) { + const host = req.body.signature.host + + waterfall([ + function loadPod (callback) { + db.Pod.loadByHost(host, callback) + }, + + function deletePod (pod, callback) { + pod.destroy().asCallback(callback) + } + ], function (err) { + if (err) return next(err) + + return res.type('json').status(204).end() + }) +} diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js deleted file mode 100644 index e54793628..000000000 --- a/server/controllers/api/remote/videos.js +++ /dev/null @@ -1,509 +0,0 @@ -'use strict' - -const eachSeries = require('async/eachSeries') -const express = require('express') -const waterfall = require('async/waterfall') - -const db = require('../../../initializers/database') -const constants = require('../../../initializers/constants') -const middlewares = require('../../../middlewares') -const secureMiddleware = middlewares.secure -const videosValidators = middlewares.validators.remote.videos -const signatureValidators = middlewares.validators.remote.signature -const logger = require('../../../helpers/logger') -const friends = require('../../../lib/friends') -const databaseUtils = require('../../../helpers/database-utils') - -const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS] - -// Functions to call when processing a remote request -const functionsHash = {} -functionsHash[ENDPOINT_ACTIONS.ADD] = addRemoteVideoRetryWrapper -functionsHash[ENDPOINT_ACTIONS.UPDATE] = updateRemoteVideoRetryWrapper -functionsHash[ENDPOINT_ACTIONS.REMOVE] = removeRemoteVideo -functionsHash[ENDPOINT_ACTIONS.REPORT_ABUSE] = reportAbuseRemoteVideo - -const router = express.Router() - -router.post('/', - signatureValidators.signature, - secureMiddleware.checkSignature, - videosValidators.remoteVideos, - remoteVideos -) - -router.post('/qadu', - signatureValidators.signature, - secureMiddleware.checkSignature, - videosValidators.remoteQaduVideos, - remoteVideosQadu -) - -router.post('/events', - signatureValidators.signature, - secureMiddleware.checkSignature, - videosValidators.remoteEventsVideos, - remoteVideosEvents -) - -// --------------------------------------------------------------------------- - -module.exports = router - -// --------------------------------------------------------------------------- - -function remoteVideos (req, res, next) { - const requests = req.body.data - const fromPod = res.locals.secure.pod - - // We need to process in the same order to keep consistency - // TODO: optimization - eachSeries(requests, function (request, callbackEach) { - const data = request.data - - // Get the function we need to call in order to process the request - const fun = functionsHash[request.type] - if (fun === undefined) { - logger.error('Unkown remote request type %s.', request.type) - return callbackEach(null) - } - - fun.call(this, data, fromPod, callbackEach) - }, function (err) { - if (err) logger.error('Error managing remote videos.', { error: err }) - }) - - // We don't need to keep the other pod waiting - return res.type('json').status(204).end() -} - -function remoteVideosQadu (req, res, next) { - const requests = req.body.data - const fromPod = res.locals.secure.pod - - eachSeries(requests, function (request, callbackEach) { - const videoData = request.data - - quickAndDirtyUpdateVideoRetryWrapper(videoData, fromPod, callbackEach) - }, function (err) { - if (err) logger.error('Error managing remote videos.', { error: err }) - }) - - return res.type('json').status(204).end() -} - -function remoteVideosEvents (req, res, next) { - const requests = req.body.data - const fromPod = res.locals.secure.pod - - eachSeries(requests, function (request, callbackEach) { - const eventData = request.data - - processVideosEventsRetryWrapper(eventData, fromPod, callbackEach) - }, function (err) { - if (err) logger.error('Error managing remote videos.', { error: err }) - }) - - return res.type('json').status(204).end() -} - -function processVideosEventsRetryWrapper (eventData, fromPod, finalCallback) { - const options = { - arguments: [ eventData, fromPod ], - errorMessage: 'Cannot process videos events with many retries.' - } - - databaseUtils.retryTransactionWrapper(processVideosEvents, options, finalCallback) -} - -function processVideosEvents (eventData, fromPod, finalCallback) { - waterfall([ - databaseUtils.startSerializableTransaction, - - function findVideo (t, callback) { - fetchOwnedVideo(eventData.remoteId, function (err, videoInstance) { - return callback(err, t, videoInstance) - }) - }, - - function updateVideoIntoDB (t, videoInstance, callback) { - const options = { transaction: t } - - let columnToUpdate - let qaduType - - switch (eventData.eventType) { - case constants.REQUEST_VIDEO_EVENT_TYPES.VIEWS: - columnToUpdate = 'views' - qaduType = constants.REQUEST_VIDEO_QADU_TYPES.VIEWS - break - - case constants.REQUEST_VIDEO_EVENT_TYPES.LIKES: - columnToUpdate = 'likes' - qaduType = constants.REQUEST_VIDEO_QADU_TYPES.LIKES - break - - case constants.REQUEST_VIDEO_EVENT_TYPES.DISLIKES: - columnToUpdate = 'dislikes' - qaduType = constants.REQUEST_VIDEO_QADU_TYPES.DISLIKES - break - - default: - return callback(new Error('Unknown video event type.')) - } - - const query = {} - query[columnToUpdate] = eventData.count - - videoInstance.increment(query, options).asCallback(function (err) { - return callback(err, t, videoInstance, qaduType) - }) - }, - - function sendQaduToFriends (t, videoInstance, qaduType, callback) { - const qadusParams = [ - { - videoId: videoInstance.id, - type: qaduType - } - ] - - friends.quickAndDirtyUpdatesVideoToFriends(qadusParams, t, function (err) { - return callback(err, t) - }) - }, - - databaseUtils.commitTransaction - - ], function (err, t) { - if (err) { - logger.debug('Cannot process a video event.', { error: err }) - return databaseUtils.rollbackTransaction(err, t, finalCallback) - } - - logger.info('Remote video event processed for video %s.', eventData.remoteId) - return finalCallback(null) - }) -} - -function quickAndDirtyUpdateVideoRetryWrapper (videoData, fromPod, finalCallback) { - const options = { - arguments: [ videoData, fromPod ], - errorMessage: 'Cannot update quick and dirty the remote video with many retries.' - } - - databaseUtils.retryTransactionWrapper(quickAndDirtyUpdateVideo, options, finalCallback) -} - -function quickAndDirtyUpdateVideo (videoData, fromPod, finalCallback) { - let videoName - - waterfall([ - databaseUtils.startSerializableTransaction, - - function findVideo (t, callback) { - fetchRemoteVideo(fromPod.host, videoData.remoteId, function (err, videoInstance) { - return callback(err, t, videoInstance) - }) - }, - - function updateVideoIntoDB (t, videoInstance, callback) { - const options = { transaction: t } - - videoName = videoInstance.name - - if (videoData.views) { - videoInstance.set('views', videoData.views) - } - - if (videoData.likes) { - videoInstance.set('likes', videoData.likes) - } - - if (videoData.dislikes) { - videoInstance.set('dislikes', videoData.dislikes) - } - - videoInstance.save(options).asCallback(function (err) { - return callback(err, t) - }) - }, - - databaseUtils.commitTransaction - - ], function (err, t) { - if (err) { - logger.debug('Cannot quick and dirty update the remote video.', { error: err }) - return databaseUtils.rollbackTransaction(err, t, finalCallback) - } - - logger.info('Remote video %s quick and dirty updated', videoName) - return finalCallback(null) - }) -} - -// Handle retries on fail -function addRemoteVideoRetryWrapper (videoToCreateData, fromPod, finalCallback) { - const options = { - arguments: [ videoToCreateData, fromPod ], - errorMessage: 'Cannot insert the remote video with many retries.' - } - - databaseUtils.retryTransactionWrapper(addRemoteVideo, options, finalCallback) -} - -function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { - logger.debug('Adding remote video "%s".', videoToCreateData.remoteId) - - waterfall([ - - databaseUtils.startSerializableTransaction, - - function assertRemoteIdAndHostUnique (t, callback) { - db.Video.loadByHostAndRemoteId(fromPod.host, videoToCreateData.remoteId, function (err, video) { - if (err) return callback(err) - - if (video) return callback(new Error('RemoteId and host pair is not unique.')) - - return callback(null, t) - }) - }, - - function findOrCreateAuthor (t, callback) { - const name = videoToCreateData.author - const podId = fromPod.id - // This author is from another pod so we do not associate a user - const userId = null - - db.Author.findOrCreateAuthor(name, podId, userId, t, function (err, authorInstance) { - return callback(err, t, authorInstance) - }) - }, - - function findOrCreateTags (t, author, callback) { - const tags = videoToCreateData.tags - - db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { - return callback(err, t, author, tagInstances) - }) - }, - - function createVideoObject (t, author, tagInstances, callback) { - const videoData = { - name: videoToCreateData.name, - remoteId: videoToCreateData.remoteId, - extname: videoToCreateData.extname, - infoHash: videoToCreateData.infoHash, - category: videoToCreateData.category, - licence: videoToCreateData.licence, - language: videoToCreateData.language, - nsfw: videoToCreateData.nsfw, - description: videoToCreateData.description, - authorId: author.id, - duration: videoToCreateData.duration, - createdAt: videoToCreateData.createdAt, - // FIXME: updatedAt does not seems to be considered by Sequelize - updatedAt: videoToCreateData.updatedAt, - views: videoToCreateData.views, - likes: videoToCreateData.likes, - dislikes: videoToCreateData.dislikes - } - - const video = db.Video.build(videoData) - - return callback(null, t, tagInstances, video) - }, - - function generateThumbnail (t, tagInstances, video, callback) { - db.Video.generateThumbnailFromData(video, videoToCreateData.thumbnailData, function (err) { - if (err) { - logger.error('Cannot generate thumbnail from data.', { error: err }) - return callback(err) - } - - return callback(err, t, tagInstances, video) - }) - }, - - function insertVideoIntoDB (t, tagInstances, video, callback) { - const options = { - transaction: t - } - - video.save(options).asCallback(function (err, videoCreated) { - return callback(err, t, tagInstances, videoCreated) - }) - }, - - function associateTagsToVideo (t, tagInstances, video, callback) { - const options = { - transaction: t - } - - video.setTags(tagInstances, options).asCallback(function (err) { - return callback(err, t) - }) - }, - - databaseUtils.commitTransaction - - ], function (err, t) { - if (err) { - // This is just a debug because we will retry the insert - logger.debug('Cannot insert the remote video.', { error: err }) - return databaseUtils.rollbackTransaction(err, t, finalCallback) - } - - logger.info('Remote video %s inserted.', videoToCreateData.name) - return finalCallback(null) - }) -} - -// Handle retries on fail -function updateRemoteVideoRetryWrapper (videoAttributesToUpdate, fromPod, finalCallback) { - const options = { - arguments: [ videoAttributesToUpdate, fromPod ], - errorMessage: 'Cannot update the remote video with many retries' - } - - databaseUtils.retryTransactionWrapper(updateRemoteVideo, options, finalCallback) -} - -function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { - logger.debug('Updating remote video "%s".', videoAttributesToUpdate.remoteId) - - waterfall([ - - databaseUtils.startSerializableTransaction, - - function findVideo (t, callback) { - fetchRemoteVideo(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) { - return callback(err, t, videoInstance) - }) - }, - - function findOrCreateTags (t, videoInstance, callback) { - const tags = videoAttributesToUpdate.tags - - db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { - return callback(err, t, videoInstance, tagInstances) - }) - }, - - function updateVideoIntoDB (t, videoInstance, tagInstances, callback) { - const options = { transaction: t } - - videoInstance.set('name', videoAttributesToUpdate.name) - videoInstance.set('category', videoAttributesToUpdate.category) - videoInstance.set('licence', videoAttributesToUpdate.licence) - videoInstance.set('language', videoAttributesToUpdate.language) - videoInstance.set('nsfw', videoAttributesToUpdate.nsfw) - videoInstance.set('description', videoAttributesToUpdate.description) - videoInstance.set('infoHash', videoAttributesToUpdate.infoHash) - videoInstance.set('duration', videoAttributesToUpdate.duration) - videoInstance.set('createdAt', videoAttributesToUpdate.createdAt) - videoInstance.set('updatedAt', videoAttributesToUpdate.updatedAt) - videoInstance.set('extname', videoAttributesToUpdate.extname) - videoInstance.set('views', videoAttributesToUpdate.views) - videoInstance.set('likes', videoAttributesToUpdate.likes) - videoInstance.set('dislikes', videoAttributesToUpdate.dislikes) - - videoInstance.save(options).asCallback(function (err) { - return callback(err, t, videoInstance, tagInstances) - }) - }, - - function associateTagsToVideo (t, videoInstance, tagInstances, callback) { - const options = { transaction: t } - - videoInstance.setTags(tagInstances, options).asCallback(function (err) { - return callback(err, t) - }) - }, - - databaseUtils.commitTransaction - - ], function (err, t) { - if (err) { - // This is just a debug because we will retry the insert - logger.debug('Cannot update the remote video.', { error: err }) - return databaseUtils.rollbackTransaction(err, t, finalCallback) - } - - logger.info('Remote video %s updated', videoAttributesToUpdate.name) - return finalCallback(null) - }) -} - -function removeRemoteVideo (videoToRemoveData, fromPod, callback) { - // We need the instance because we have to remove some other stuffs (thumbnail etc) - fetchRemoteVideo(fromPod.host, videoToRemoveData.remoteId, function (err, video) { - // Do not return the error, continue the process - if (err) return callback(null) - - logger.debug('Removing remote video %s.', video.remoteId) - video.destroy().asCallback(function (err) { - // Do not return the error, continue the process - if (err) { - logger.error('Cannot remove remote video with id %s.', videoToRemoveData.remoteId, { error: err }) - } - - return callback(null) - }) - }) -} - -function reportAbuseRemoteVideo (reportData, fromPod, callback) { - fetchOwnedVideo(reportData.videoRemoteId, function (err, video) { - if (err || !video) { - if (!err) err = new Error('video not found') - - logger.error('Cannot load video from id.', { error: err, id: reportData.videoRemoteId }) - // Do not return the error, continue the process - return callback(null) - } - - logger.debug('Reporting remote abuse for video %s.', video.id) - - const videoAbuseData = { - reporterUsername: reportData.reporterUsername, - reason: reportData.reportReason, - reporterPodId: fromPod.id, - videoId: video.id - } - - db.VideoAbuse.create(videoAbuseData).asCallback(function (err) { - if (err) { - logger.error('Cannot create remote abuse video.', { error: err }) - } - - return callback(null) - }) - }) -} - -function fetchOwnedVideo (id, callback) { - db.Video.load(id, function (err, video) { - if (err || !video) { - if (!err) err = new Error('video not found') - - logger.error('Cannot load owned video from id.', { error: err, id }) - return callback(err) - } - - return callback(null, video) - }) -} - -function fetchRemoteVideo (podHost, remoteId, callback) { - db.Video.loadByHostAndRemoteId(podHost, remoteId, function (err, video) { - if (err || !video) { - if (!err) err = new Error('video not found') - - logger.error('Cannot load video from host and remote id.', { error: err, podHost, remoteId }) - return callback(err) - } - - return callback(null, video) - }) -} diff --git a/server/controllers/api/remote/videos.ts b/server/controllers/api/remote/videos.ts new file mode 100644 index 000000000..df4ba8309 --- /dev/null +++ b/server/controllers/api/remote/videos.ts @@ -0,0 +1,521 @@ +import express = require('express') +import { eachSeries, waterfall } from 'async' + +const db = require('../../../initializers/database') +import { + REQUEST_ENDPOINT_ACTIONS, + REQUEST_ENDPOINTS, + REQUEST_VIDEO_EVENT_TYPES, + REQUEST_VIDEO_QADU_TYPES +} from '../../../initializers' +import { + checkSignature, + signatureValidator, + remoteVideosValidator, + remoteQaduVideosValidator, + remoteEventsVideosValidator +} from '../../../middlewares' +import { + logger, + commitTransaction, + retryTransactionWrapper, + rollbackTransaction, + startSerializableTransaction +} from '../../../helpers' +import { quickAndDirtyUpdatesVideoToFriends } from '../../../lib' + +const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] + +// Functions to call when processing a remote request +const functionsHash = {} +functionsHash[ENDPOINT_ACTIONS.ADD] = addRemoteVideoRetryWrapper +functionsHash[ENDPOINT_ACTIONS.UPDATE] = updateRemoteVideoRetryWrapper +functionsHash[ENDPOINT_ACTIONS.REMOVE] = removeRemoteVideo +functionsHash[ENDPOINT_ACTIONS.REPORT_ABUSE] = reportAbuseRemoteVideo + +const remoteVideosRouter = express.Router() + +remoteVideosRouter.post('/', + signatureValidator, + checkSignature, + remoteVideosValidator, + remoteVideos +) + +remoteVideosRouter.post('/qadu', + signatureValidator, + checkSignature, + remoteQaduVideosValidator, + remoteVideosQadu +) + +remoteVideosRouter.post('/events', + signatureValidator, + checkSignature, + remoteEventsVideosValidator, + remoteVideosEvents +) + +// --------------------------------------------------------------------------- + +export { + remoteVideosRouter +} + +// --------------------------------------------------------------------------- + +function remoteVideos (req, res, next) { + const requests = req.body.data + const fromPod = res.locals.secure.pod + + // We need to process in the same order to keep consistency + // TODO: optimization + eachSeries(requests, function (request: any, callbackEach) { + const data = request.data + + // Get the function we need to call in order to process the request + const fun = functionsHash[request.type] + if (fun === undefined) { + logger.error('Unkown remote request type %s.', request.type) + return callbackEach(null) + } + + fun.call(this, data, fromPod, callbackEach) + }, function (err) { + if (err) logger.error('Error managing remote videos.', { error: err }) + }) + + // We don't need to keep the other pod waiting + return res.type('json').status(204).end() +} + +function remoteVideosQadu (req, res, next) { + const requests = req.body.data + const fromPod = res.locals.secure.pod + + eachSeries(requests, function (request: any, callbackEach) { + const videoData = request.data + + quickAndDirtyUpdateVideoRetryWrapper(videoData, fromPod, callbackEach) + }, function (err) { + if (err) logger.error('Error managing remote videos.', { error: err }) + }) + + return res.type('json').status(204).end() +} + +function remoteVideosEvents (req, res, next) { + const requests = req.body.data + const fromPod = res.locals.secure.pod + + eachSeries(requests, function (request: any, callbackEach) { + const eventData = request.data + + processVideosEventsRetryWrapper(eventData, fromPod, callbackEach) + }, function (err) { + if (err) logger.error('Error managing remote videos.', { error: err }) + }) + + return res.type('json').status(204).end() +} + +function processVideosEventsRetryWrapper (eventData, fromPod, finalCallback) { + const options = { + arguments: [ eventData, fromPod ], + errorMessage: 'Cannot process videos events with many retries.' + } + + retryTransactionWrapper(processVideosEvents, options, finalCallback) +} + +function processVideosEvents (eventData, fromPod, finalCallback) { + waterfall([ + startSerializableTransaction, + + function findVideo (t, callback) { + fetchOwnedVideo(eventData.remoteId, function (err, videoInstance) { + return callback(err, t, videoInstance) + }) + }, + + function updateVideoIntoDB (t, videoInstance, callback) { + const options = { transaction: t } + + let columnToUpdate + let qaduType + + switch (eventData.eventType) { + case REQUEST_VIDEO_EVENT_TYPES.VIEWS: + columnToUpdate = 'views' + qaduType = REQUEST_VIDEO_QADU_TYPES.VIEWS + break + + case REQUEST_VIDEO_EVENT_TYPES.LIKES: + columnToUpdate = 'likes' + qaduType = REQUEST_VIDEO_QADU_TYPES.LIKES + break + + case REQUEST_VIDEO_EVENT_TYPES.DISLIKES: + columnToUpdate = 'dislikes' + qaduType = REQUEST_VIDEO_QADU_TYPES.DISLIKES + break + + default: + return callback(new Error('Unknown video event type.')) + } + + const query = {} + query[columnToUpdate] = eventData.count + + videoInstance.increment(query, options).asCallback(function (err) { + return callback(err, t, videoInstance, qaduType) + }) + }, + + function sendQaduToFriends (t, videoInstance, qaduType, callback) { + const qadusParams = [ + { + videoId: videoInstance.id, + type: qaduType + } + ] + + quickAndDirtyUpdatesVideoToFriends(qadusParams, t, function (err) { + return callback(err, t) + }) + }, + + commitTransaction + + ], function (err, t) { + if (err) { + logger.debug('Cannot process a video event.', { error: err }) + return rollbackTransaction(err, t, finalCallback) + } + + logger.info('Remote video event processed for video %s.', eventData.remoteId) + return finalCallback(null) + }) +} + +function quickAndDirtyUpdateVideoRetryWrapper (videoData, fromPod, finalCallback) { + const options = { + arguments: [ videoData, fromPod ], + errorMessage: 'Cannot update quick and dirty the remote video with many retries.' + } + + retryTransactionWrapper(quickAndDirtyUpdateVideo, options, finalCallback) +} + +function quickAndDirtyUpdateVideo (videoData, fromPod, finalCallback) { + let videoName + + waterfall([ + startSerializableTransaction, + + function findVideo (t, callback) { + fetchRemoteVideo(fromPod.host, videoData.remoteId, function (err, videoInstance) { + return callback(err, t, videoInstance) + }) + }, + + function updateVideoIntoDB (t, videoInstance, callback) { + const options = { transaction: t } + + videoName = videoInstance.name + + if (videoData.views) { + videoInstance.set('views', videoData.views) + } + + if (videoData.likes) { + videoInstance.set('likes', videoData.likes) + } + + if (videoData.dislikes) { + videoInstance.set('dislikes', videoData.dislikes) + } + + videoInstance.save(options).asCallback(function (err) { + return callback(err, t) + }) + }, + + commitTransaction + + ], function (err, t) { + if (err) { + logger.debug('Cannot quick and dirty update the remote video.', { error: err }) + return rollbackTransaction(err, t, finalCallback) + } + + logger.info('Remote video %s quick and dirty updated', videoName) + return finalCallback(null) + }) +} + +// Handle retries on fail +function addRemoteVideoRetryWrapper (videoToCreateData, fromPod, finalCallback) { + const options = { + arguments: [ videoToCreateData, fromPod ], + errorMessage: 'Cannot insert the remote video with many retries.' + } + + retryTransactionWrapper(addRemoteVideo, options, finalCallback) +} + +function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { + logger.debug('Adding remote video "%s".', videoToCreateData.remoteId) + + waterfall([ + + startSerializableTransaction, + + function assertRemoteIdAndHostUnique (t, callback) { + db.Video.loadByHostAndRemoteId(fromPod.host, videoToCreateData.remoteId, function (err, video) { + if (err) return callback(err) + + if (video) return callback(new Error('RemoteId and host pair is not unique.')) + + return callback(null, t) + }) + }, + + function findOrCreateAuthor (t, callback) { + const name = videoToCreateData.author + const podId = fromPod.id + // This author is from another pod so we do not associate a user + const userId = null + + db.Author.findOrCreateAuthor(name, podId, userId, t, function (err, authorInstance) { + return callback(err, t, authorInstance) + }) + }, + + function findOrCreateTags (t, author, callback) { + const tags = videoToCreateData.tags + + db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { + return callback(err, t, author, tagInstances) + }) + }, + + function createVideoObject (t, author, tagInstances, callback) { + const videoData = { + name: videoToCreateData.name, + remoteId: videoToCreateData.remoteId, + extname: videoToCreateData.extname, + infoHash: videoToCreateData.infoHash, + category: videoToCreateData.category, + licence: videoToCreateData.licence, + language: videoToCreateData.language, + nsfw: videoToCreateData.nsfw, + description: videoToCreateData.description, + authorId: author.id, + duration: videoToCreateData.duration, + createdAt: videoToCreateData.createdAt, + // FIXME: updatedAt does not seems to be considered by Sequelize + updatedAt: videoToCreateData.updatedAt, + views: videoToCreateData.views, + likes: videoToCreateData.likes, + dislikes: videoToCreateData.dislikes + } + + const video = db.Video.build(videoData) + + return callback(null, t, tagInstances, video) + }, + + function generateThumbnail (t, tagInstances, video, callback) { + db.Video.generateThumbnailFromData(video, videoToCreateData.thumbnailData, function (err) { + if (err) { + logger.error('Cannot generate thumbnail from data.', { error: err }) + return callback(err) + } + + return callback(err, t, tagInstances, video) + }) + }, + + function insertVideoIntoDB (t, tagInstances, video, callback) { + const options = { + transaction: t + } + + video.save(options).asCallback(function (err, videoCreated) { + return callback(err, t, tagInstances, videoCreated) + }) + }, + + function associateTagsToVideo (t, tagInstances, video, callback) { + const options = { + transaction: t + } + + video.setTags(tagInstances, options).asCallback(function (err) { + return callback(err, t) + }) + }, + + commitTransaction + + ], function (err, t) { + if (err) { + // This is just a debug because we will retry the insert + logger.debug('Cannot insert the remote video.', { error: err }) + return rollbackTransaction(err, t, finalCallback) + } + + logger.info('Remote video %s inserted.', videoToCreateData.name) + return finalCallback(null) + }) +} + +// Handle retries on fail +function updateRemoteVideoRetryWrapper (videoAttributesToUpdate, fromPod, finalCallback) { + const options = { + arguments: [ videoAttributesToUpdate, fromPod ], + errorMessage: 'Cannot update the remote video with many retries' + } + + retryTransactionWrapper(updateRemoteVideo, options, finalCallback) +} + +function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { + logger.debug('Updating remote video "%s".', videoAttributesToUpdate.remoteId) + + waterfall([ + + startSerializableTransaction, + + function findVideo (t, callback) { + fetchRemoteVideo(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) { + return callback(err, t, videoInstance) + }) + }, + + function findOrCreateTags (t, videoInstance, callback) { + const tags = videoAttributesToUpdate.tags + + db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { + return callback(err, t, videoInstance, tagInstances) + }) + }, + + function updateVideoIntoDB (t, videoInstance, tagInstances, callback) { + const options = { transaction: t } + + videoInstance.set('name', videoAttributesToUpdate.name) + videoInstance.set('category', videoAttributesToUpdate.category) + videoInstance.set('licence', videoAttributesToUpdate.licence) + videoInstance.set('language', videoAttributesToUpdate.language) + videoInstance.set('nsfw', videoAttributesToUpdate.nsfw) + videoInstance.set('description', videoAttributesToUpdate.description) + videoInstance.set('infoHash', videoAttributesToUpdate.infoHash) + videoInstance.set('duration', videoAttributesToUpdate.duration) + videoInstance.set('createdAt', videoAttributesToUpdate.createdAt) + videoInstance.set('updatedAt', videoAttributesToUpdate.updatedAt) + videoInstance.set('extname', videoAttributesToUpdate.extname) + videoInstance.set('views', videoAttributesToUpdate.views) + videoInstance.set('likes', videoAttributesToUpdate.likes) + videoInstance.set('dislikes', videoAttributesToUpdate.dislikes) + + videoInstance.save(options).asCallback(function (err) { + return callback(err, t, videoInstance, tagInstances) + }) + }, + + function associateTagsToVideo (t, videoInstance, tagInstances, callback) { + const options = { transaction: t } + + videoInstance.setTags(tagInstances, options).asCallback(function (err) { + return callback(err, t) + }) + }, + + commitTransaction + + ], function (err, t) { + if (err) { + // This is just a debug because we will retry the insert + logger.debug('Cannot update the remote video.', { error: err }) + return rollbackTransaction(err, t, finalCallback) + } + + logger.info('Remote video %s updated', videoAttributesToUpdate.name) + return finalCallback(null) + }) +} + +function removeRemoteVideo (videoToRemoveData, fromPod, callback) { + // We need the instance because we have to remove some other stuffs (thumbnail etc) + fetchRemoteVideo(fromPod.host, videoToRemoveData.remoteId, function (err, video) { + // Do not return the error, continue the process + if (err) return callback(null) + + logger.debug('Removing remote video %s.', video.remoteId) + video.destroy().asCallback(function (err) { + // Do not return the error, continue the process + if (err) { + logger.error('Cannot remove remote video with id %s.', videoToRemoveData.remoteId, { error: err }) + } + + return callback(null) + }) + }) +} + +function reportAbuseRemoteVideo (reportData, fromPod, callback) { + fetchOwnedVideo(reportData.videoRemoteId, function (err, video) { + if (err || !video) { + if (!err) err = new Error('video not found') + + logger.error('Cannot load video from id.', { error: err, id: reportData.videoRemoteId }) + // Do not return the error, continue the process + return callback(null) + } + + logger.debug('Reporting remote abuse for video %s.', video.id) + + const videoAbuseData = { + reporterUsername: reportData.reporterUsername, + reason: reportData.reportReason, + reporterPodId: fromPod.id, + videoId: video.id + } + + db.VideoAbuse.create(videoAbuseData).asCallback(function (err) { + if (err) { + logger.error('Cannot create remote abuse video.', { error: err }) + } + + return callback(null) + }) + }) +} + +function fetchOwnedVideo (id, callback) { + db.Video.load(id, function (err, video) { + if (err || !video) { + if (!err) err = new Error('video not found') + + logger.error('Cannot load owned video from id.', { error: err, id }) + return callback(err) + } + + return callback(null, video) + }) +} + +function fetchRemoteVideo (podHost, remoteId, callback) { + db.Video.loadByHostAndRemoteId(podHost, remoteId, function (err, video) { + if (err || !video) { + if (!err) err = new Error('video not found') + + logger.error('Cannot load video from host and remote id.', { error: err, podHost, remoteId }) + return callback(err) + } + + return callback(null, video) + }) +} diff --git a/server/controllers/api/requests.js b/server/controllers/api/requests.js deleted file mode 100644 index 6fd5753ac..000000000 --- a/server/controllers/api/requests.js +++ /dev/null @@ -1,55 +0,0 @@ -'use strict' - -const express = require('express') -const parallel = require('async/parallel') - -const friends = require('../../lib/friends') -const middlewares = require('../../middlewares') -const admin = middlewares.admin -const oAuth = middlewares.oauth - -const router = express.Router() - -router.get('/stats', - oAuth.authenticate, - admin.ensureIsAdmin, - getStatsRequests -) - -// --------------------------------------------------------------------------- - -module.exports = router - -// --------------------------------------------------------------------------- - -function getStatsRequests (req, res, next) { - parallel({ - requestScheduler: buildRequestSchedulerFunction(friends.getRequestScheduler()), - requestVideoQaduScheduler: buildRequestSchedulerFunction(friends.getRequestVideoQaduScheduler()), - requestVideoEventScheduler: buildRequestSchedulerFunction(friends.getRequestVideoEventScheduler()) - }, function (err, result) { - if (err) return next(err) - - return res.json(result) - }) -} - -// --------------------------------------------------------------------------- - -function buildRequestSchedulerFunction (requestScheduler) { - return function (callback) { - requestScheduler.remainingRequestsCount(function (err, count) { - if (err) return callback(err) - - const result = { - totalRequests: count, - requestsLimitPods: requestScheduler.limitPods, - requestsLimitPerPod: requestScheduler.limitPerPod, - remainingMilliSeconds: requestScheduler.remainingMilliSeconds(), - milliSecondsInterval: requestScheduler.requestInterval - } - - return callback(null, result) - }) - } -} diff --git a/server/controllers/api/requests.ts b/server/controllers/api/requests.ts new file mode 100644 index 000000000..304499a4f --- /dev/null +++ b/server/controllers/api/requests.ts @@ -0,0 +1,57 @@ +import express = require('express') +import { parallel } from 'async' + +import { + getRequestScheduler, + getRequestVideoQaduScheduler, + getRequestVideoEventScheduler +} from '../../lib' +import { authenticate, ensureIsAdmin } from '../../middlewares' + +const requestsRouter = express.Router() + +requestsRouter.get('/stats', + authenticate, + ensureIsAdmin, + getStatsRequests +) + +// --------------------------------------------------------------------------- + +export { + requestsRouter +} + +// --------------------------------------------------------------------------- + +function getStatsRequests (req, res, next) { + parallel({ + requestScheduler: buildRequestSchedulerFunction(getRequestScheduler()), + requestVideoQaduScheduler: buildRequestSchedulerFunction(getRequestVideoQaduScheduler()), + requestVideoEventScheduler: buildRequestSchedulerFunction(getRequestVideoEventScheduler()) + }, function (err, result) { + if (err) return next(err) + + return res.json(result) + }) +} + +// --------------------------------------------------------------------------- + +function buildRequestSchedulerFunction (requestScheduler) { + return function (callback) { + requestScheduler.remainingRequestsCount(function (err, count) { + if (err) return callback(err) + + const result = { + totalRequests: count, + requestsLimitPods: requestScheduler.limitPods, + requestsLimitPerPod: requestScheduler.limitPerPod, + remainingMilliSeconds: requestScheduler.remainingMilliSeconds(), + milliSecondsInterval: requestScheduler.requestInterval + } + + return callback(null, result) + }) + } +} diff --git a/server/controllers/api/users.js b/server/controllers/api/users.js deleted file mode 100644 index c7fe7bf85..000000000 --- a/server/controllers/api/users.js +++ /dev/null @@ -1,169 +0,0 @@ -'use strict' - -const express = require('express') -const waterfall = require('async/waterfall') - -const constants = require('../../initializers/constants') -const db = require('../../initializers/database') -const logger = require('../../helpers/logger') -const utils = require('../../helpers/utils') -const middlewares = require('../../middlewares') -const admin = middlewares.admin -const oAuth = middlewares.oauth -const pagination = middlewares.pagination -const sort = middlewares.sort -const validatorsPagination = middlewares.validators.pagination -const validatorsSort = middlewares.validators.sort -const validatorsUsers = middlewares.validators.users - -const router = express.Router() - -router.get('/me', - oAuth.authenticate, - getUserInformation -) - -router.get('/me/videos/:videoId/rating', - oAuth.authenticate, - validatorsUsers.usersVideoRating, - getUserVideoRating -) - -router.get('/', - validatorsPagination.pagination, - validatorsSort.usersSort, - sort.setUsersSort, - pagination.setPagination, - listUsers -) - -router.post('/', - oAuth.authenticate, - admin.ensureIsAdmin, - validatorsUsers.usersAdd, - createUser -) - -router.post('/register', - ensureRegistrationEnabled, - validatorsUsers.usersAdd, - createUser -) - -router.put('/:id', - oAuth.authenticate, - validatorsUsers.usersUpdate, - updateUser -) - -router.delete('/:id', - oAuth.authenticate, - admin.ensureIsAdmin, - validatorsUsers.usersRemove, - removeUser -) - -router.post('/token', oAuth.token, success) -// TODO: Once https://github.com/oauthjs/node-oauth2-server/pull/289 is merged, implement revoke token route - -// --------------------------------------------------------------------------- - -module.exports = router - -// --------------------------------------------------------------------------- - -function ensureRegistrationEnabled (req, res, next) { - const registrationEnabled = constants.CONFIG.SIGNUP.ENABLED - - if (registrationEnabled === true) { - return next() - } - - return res.status(400).send('User registration is not enabled.') -} - -function createUser (req, res, next) { - const user = db.User.build({ - username: req.body.username, - password: req.body.password, - email: req.body.email, - displayNSFW: false, - role: constants.USER_ROLES.USER - }) - - user.save().asCallback(function (err, createdUser) { - if (err) return next(err) - - return res.type('json').status(204).end() - }) -} - -function getUserInformation (req, res, next) { - db.User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) { - if (err) return next(err) - - return res.json(user.toFormatedJSON()) - }) -} - -function getUserVideoRating (req, res, next) { - const videoId = req.params.videoId - const userId = res.locals.oauth.token.User.id - - db.UserVideoRate.load(userId, videoId, function (err, ratingObj) { - if (err) return next(err) - - const rating = ratingObj ? ratingObj.type : 'none' - - res.json({ - videoId, - rating - }) - }) -} - -function listUsers (req, res, next) { - db.User.listForApi(req.query.start, req.query.count, req.query.sort, function (err, usersList, usersTotal) { - if (err) return next(err) - - res.json(utils.getFormatedObjects(usersList, usersTotal)) - }) -} - -function removeUser (req, res, next) { - waterfall([ - function loadUser (callback) { - db.User.loadById(req.params.id, callback) - }, - - function deleteUser (user, callback) { - user.destroy().asCallback(callback) - } - ], function andFinally (err) { - if (err) { - logger.error('Errors when removed the user.', { error: err }) - return next(err) - } - - return res.sendStatus(204) - }) -} - -function updateUser (req, res, next) { - db.User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) { - if (err) return next(err) - - if (req.body.password) user.password = req.body.password - if (req.body.displayNSFW !== undefined) user.displayNSFW = req.body.displayNSFW - - user.save().asCallback(function (err) { - if (err) return next(err) - - return res.sendStatus(204) - }) - }) -} - -function success (req, res, next) { - res.end() -} diff --git a/server/controllers/api/users.ts b/server/controllers/api/users.ts new file mode 100644 index 000000000..981a4706a --- /dev/null +++ b/server/controllers/api/users.ts @@ -0,0 +1,173 @@ +import express = require('express') +import { waterfall } from 'async' + +const db = require('../../initializers/database') +import { CONFIG, USER_ROLES } from '../../initializers' +import { logger, getFormatedObjects } from '../../helpers' +import { + authenticate, + ensureIsAdmin, + usersAddValidator, + usersUpdateValidator, + usersRemoveValidator, + usersVideoRatingValidator, + paginationValidator, + setPagination, + usersSortValidator, + setUsersSort, + token +} from '../../middlewares' + +const usersRouter = express.Router() + +usersRouter.get('/me', + authenticate, + getUserInformation +) + +usersRouter.get('/me/videos/:videoId/rating', + authenticate, + usersVideoRatingValidator, + getUserVideoRating +) + +usersRouter.get('/', + paginationValidator, + usersSortValidator, + setUsersSort, + setPagination, + listUsers +) + +usersRouter.post('/', + authenticate, + ensureIsAdmin, + usersAddValidator, + createUser +) + +usersRouter.post('/register', + ensureRegistrationEnabled, + usersAddValidator, + createUser +) + +usersRouter.put('/:id', + authenticate, + usersUpdateValidator, + updateUser +) + +usersRouter.delete('/:id', + authenticate, + ensureIsAdmin, + usersRemoveValidator, + removeUser +) + +usersRouter.post('/token', token, success) +// TODO: Once https://github.com/oauthjs/node-oauth2-server/pull/289 is merged, implement revoke token route + +// --------------------------------------------------------------------------- + +export { + usersRouter +} + +// --------------------------------------------------------------------------- + +function ensureRegistrationEnabled (req, res, next) { + const registrationEnabled = CONFIG.SIGNUP.ENABLED + + if (registrationEnabled === true) { + return next() + } + + return res.status(400).send('User registration is not enabled.') +} + +function createUser (req, res, next) { + const user = db.User.build({ + username: req.body.username, + password: req.body.password, + email: req.body.email, + displayNSFW: false, + role: USER_ROLES.USER + }) + + user.save().asCallback(function (err, createdUser) { + if (err) return next(err) + + return res.type('json').status(204).end() + }) +} + +function getUserInformation (req, res, next) { + db.User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) { + if (err) return next(err) + + return res.json(user.toFormatedJSON()) + }) +} + +function getUserVideoRating (req, res, next) { + const videoId = req.params.videoId + const userId = res.locals.oauth.token.User.id + + db.UserVideoRate.load(userId, videoId, function (err, ratingObj) { + if (err) return next(err) + + const rating = ratingObj ? ratingObj.type : 'none' + + res.json({ + videoId, + rating + }) + }) +} + +function listUsers (req, res, next) { + db.User.listForApi(req.query.start, req.query.count, req.query.sort, function (err, usersList, usersTotal) { + if (err) return next(err) + + res.json(getFormatedObjects(usersList, usersTotal)) + }) +} + +function removeUser (req, res, next) { + waterfall([ + function loadUser (callback) { + db.User.loadById(req.params.id, callback) + }, + + function deleteUser (user, callback) { + user.destroy().asCallback(callback) + } + ], function andFinally (err) { + if (err) { + logger.error('Errors when removed the user.', { error: err }) + return next(err) + } + + return res.sendStatus(204) + }) +} + +function updateUser (req, res, next) { + db.User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) { + if (err) return next(err) + + if (req.body.password) user.password = req.body.password + if (req.body.displayNSFW !== undefined) user.displayNSFW = req.body.displayNSFW + + user.save().asCallback(function (err) { + if (err) return next(err) + + return res.sendStatus(204) + }) + }) +} + +function success (req, res, next) { + res.end() +} diff --git a/server/controllers/api/videos/abuse.js b/server/controllers/api/videos/abuse.js deleted file mode 100644 index 0fb44bb14..000000000 --- a/server/controllers/api/videos/abuse.js +++ /dev/null @@ -1,112 +0,0 @@ -'use strict' - -const express = require('express') -const waterfall = require('async/waterfall') - -const db = require('../../../initializers/database') -const logger = require('../../../helpers/logger') -const friends = require('../../../lib/friends') -const middlewares = require('../../../middlewares') -const admin = middlewares.admin -const oAuth = middlewares.oauth -const pagination = middlewares.pagination -const validators = middlewares.validators -const validatorsPagination = validators.pagination -const validatorsSort = validators.sort -const validatorsVideos = validators.videos -const sort = middlewares.sort -const databaseUtils = require('../../../helpers/database-utils') -const utils = require('../../../helpers/utils') - -const router = express.Router() - -router.get('/abuse', - oAuth.authenticate, - admin.ensureIsAdmin, - validatorsPagination.pagination, - validatorsSort.videoAbusesSort, - sort.setVideoAbusesSort, - pagination.setPagination, - listVideoAbuses -) -router.post('/:id/abuse', - oAuth.authenticate, - validatorsVideos.videoAbuseReport, - reportVideoAbuseRetryWrapper -) - -// --------------------------------------------------------------------------- - -module.exports = router - -// --------------------------------------------------------------------------- - -function listVideoAbuses (req, res, next) { - db.VideoAbuse.listForApi(req.query.start, req.query.count, req.query.sort, function (err, abusesList, abusesTotal) { - if (err) return next(err) - - res.json(utils.getFormatedObjects(abusesList, abusesTotal)) - }) -} - -function reportVideoAbuseRetryWrapper (req, res, next) { - const options = { - arguments: [ req, res ], - errorMessage: 'Cannot report abuse to the video with many retries.' - } - - databaseUtils.retryTransactionWrapper(reportVideoAbuse, options, function (err) { - if (err) return next(err) - - return res.type('json').status(204).end() - }) -} - -function reportVideoAbuse (req, res, finalCallback) { - const videoInstance = res.locals.video - const reporterUsername = res.locals.oauth.token.User.username - - const abuse = { - reporterUsername, - reason: req.body.reason, - videoId: videoInstance.id, - reporterPodId: null // This is our pod that reported this abuse - } - - waterfall([ - - databaseUtils.startSerializableTransaction, - - function createAbuse (t, callback) { - db.VideoAbuse.create(abuse).asCallback(function (err, abuse) { - return callback(err, t, abuse) - }) - }, - - function sendToFriendsIfNeeded (t, abuse, callback) { - // We send the information to the destination pod - if (videoInstance.isOwned() === false) { - const reportData = { - reporterUsername, - reportReason: abuse.reason, - videoRemoteId: videoInstance.remoteId - } - - friends.reportAbuseVideoToFriend(reportData, videoInstance) - } - - return callback(null, t) - }, - - databaseUtils.commitTransaction - - ], function andFinally (err, t) { - if (err) { - logger.debug('Cannot update the video.', { error: err }) - return databaseUtils.rollbackTransaction(err, t, finalCallback) - } - - logger.info('Abuse report for video %s created.', videoInstance.name) - return finalCallback(null) - }) -} diff --git a/server/controllers/api/videos/abuse.ts b/server/controllers/api/videos/abuse.ts new file mode 100644 index 000000000..88204120f --- /dev/null +++ b/server/controllers/api/videos/abuse.ts @@ -0,0 +1,117 @@ +import express = require('express') +import { waterfall } from 'async' + +const db = require('../../../initializers/database') +import friends = require('../../../lib/friends') +import { + logger, + getFormatedObjects, + retryTransactionWrapper, + startSerializableTransaction, + commitTransaction, + rollbackTransaction +} from '../../../helpers' +import { + authenticate, + ensureIsAdmin, + paginationValidator, + videoAbuseReportValidator, + videoAbusesSortValidator, + setVideoAbusesSort, + setPagination +} from '../../../middlewares' + +const abuseVideoRouter = express.Router() + +abuseVideoRouter.get('/abuse', + authenticate, + ensureIsAdmin, + paginationValidator, + videoAbusesSortValidator, + setVideoAbusesSort, + setPagination, + listVideoAbuses +) +abuseVideoRouter.post('/:id/abuse', + authenticate, + videoAbuseReportValidator, + reportVideoAbuseRetryWrapper +) + +// --------------------------------------------------------------------------- + +export { + abuseVideoRouter +} + +// --------------------------------------------------------------------------- + +function listVideoAbuses (req, res, next) { + db.VideoAbuse.listForApi(req.query.start, req.query.count, req.query.sort, function (err, abusesList, abusesTotal) { + if (err) return next(err) + + res.json(getFormatedObjects(abusesList, abusesTotal)) + }) +} + +function reportVideoAbuseRetryWrapper (req, res, next) { + const options = { + arguments: [ req, res ], + errorMessage: 'Cannot report abuse to the video with many retries.' + } + + retryTransactionWrapper(reportVideoAbuse, options, function (err) { + if (err) return next(err) + + return res.type('json').status(204).end() + }) +} + +function reportVideoAbuse (req, res, finalCallback) { + const videoInstance = res.locals.video + const reporterUsername = res.locals.oauth.token.User.username + + const abuse = { + reporterUsername, + reason: req.body.reason, + videoId: videoInstance.id, + reporterPodId: null // This is our pod that reported this abuse + } + + waterfall([ + + startSerializableTransaction, + + function createAbuse (t, callback) { + db.VideoAbuse.create(abuse).asCallback(function (err, abuse) { + return callback(err, t, abuse) + }) + }, + + function sendToFriendsIfNeeded (t, abuse, callback) { + // We send the information to the destination pod + if (videoInstance.isOwned() === false) { + const reportData = { + reporterUsername, + reportReason: abuse.reason, + videoRemoteId: videoInstance.remoteId + } + + friends.reportAbuseVideoToFriend(reportData, videoInstance) + } + + return callback(null, t) + }, + + commitTransaction + + ], function andFinally (err, t) { + if (err) { + logger.debug('Cannot update the video.', { error: err }) + return rollbackTransaction(err, t, finalCallback) + } + + logger.info('Abuse report for video %s created.', videoInstance.name) + return finalCallback(null) + }) +} diff --git a/server/controllers/api/videos/blacklist.js b/server/controllers/api/videos/blacklist.js deleted file mode 100644 index 8c3e2a69d..000000000 --- a/server/controllers/api/videos/blacklist.js +++ /dev/null @@ -1,43 +0,0 @@ -'use strict' - -const express = require('express') - -const db = require('../../../initializers/database') -const logger = require('../../../helpers/logger') -const middlewares = require('../../../middlewares') -const admin = middlewares.admin -const oAuth = middlewares.oauth -const validators = middlewares.validators -const validatorsVideos = validators.videos - -const router = express.Router() - -router.post('/:id/blacklist', - oAuth.authenticate, - admin.ensureIsAdmin, - validatorsVideos.videosBlacklist, - addVideoToBlacklist -) - -// --------------------------------------------------------------------------- - -module.exports = router - -// --------------------------------------------------------------------------- - -function addVideoToBlacklist (req, res, next) { - const videoInstance = res.locals.video - - const toCreate = { - videoId: videoInstance.id - } - - db.BlacklistedVideo.create(toCreate).asCallback(function (err) { - if (err) { - logger.error('Errors when blacklisting video ', { error: err }) - return next(err) - } - - return res.type('json').status(204).end() - }) -} diff --git a/server/controllers/api/videos/blacklist.ts b/server/controllers/api/videos/blacklist.ts new file mode 100644 index 000000000..db6d95e73 --- /dev/null +++ b/server/controllers/api/videos/blacklist.ts @@ -0,0 +1,43 @@ +import express = require('express') + +const db = require('../../../initializers/database') +import { logger } from '../../../helpers' +import { + authenticate, + ensureIsAdmin, + videosBlacklistValidator +} from '../../../middlewares' + +const blacklistRouter = express.Router() + +blacklistRouter.post('/:id/blacklist', + authenticate, + ensureIsAdmin, + videosBlacklistValidator, + addVideoToBlacklist +) + +// --------------------------------------------------------------------------- + +export { + blacklistRouter +} + +// --------------------------------------------------------------------------- + +function addVideoToBlacklist (req, res, next) { + const videoInstance = res.locals.video + + const toCreate = { + videoId: videoInstance.id + } + + db.BlacklistedVideo.create(toCreate).asCallback(function (err) { + if (err) { + logger.error('Errors when blacklisting video ', { error: err }) + return next(err) + } + + return res.type('json').status(204).end() + }) +} diff --git a/server/controllers/api/videos/index.js b/server/controllers/api/videos/index.js deleted file mode 100644 index 8de44d5ac..000000000 --- a/server/controllers/api/videos/index.js +++ /dev/null @@ -1,404 +0,0 @@ -'use strict' - -const express = require('express') -const fs = require('fs') -const multer = require('multer') -const path = require('path') -const waterfall = require('async/waterfall') - -const constants = require('../../../initializers/constants') -const db = require('../../../initializers/database') -const logger = require('../../../helpers/logger') -const friends = require('../../../lib/friends') -const middlewares = require('../../../middlewares') -const oAuth = middlewares.oauth -const pagination = middlewares.pagination -const validators = middlewares.validators -const validatorsPagination = validators.pagination -const validatorsSort = validators.sort -const validatorsVideos = validators.videos -const search = middlewares.search -const sort = middlewares.sort -const databaseUtils = require('../../../helpers/database-utils') -const utils = require('../../../helpers/utils') - -const abuseController = require('./abuse') -const blacklistController = require('./blacklist') -const rateController = require('./rate') - -const router = express.Router() - -// multer configuration -const storage = multer.diskStorage({ - destination: function (req, file, cb) { - cb(null, constants.CONFIG.STORAGE.VIDEOS_DIR) - }, - - filename: function (req, file, cb) { - let extension = '' - if (file.mimetype === 'video/webm') extension = 'webm' - else if (file.mimetype === 'video/mp4') extension = 'mp4' - else if (file.mimetype === 'video/ogg') extension = 'ogv' - utils.generateRandomString(16, function (err, randomString) { - const fieldname = err ? undefined : randomString - cb(null, fieldname + '.' + extension) - }) - } -}) - -const reqFiles = multer({ storage: storage }).fields([{ name: 'videofile', maxCount: 1 }]) - -router.use('/', abuseController) -router.use('/', blacklistController) -router.use('/', rateController) - -router.get('/categories', listVideoCategories) -router.get('/licences', listVideoLicences) -router.get('/languages', listVideoLanguages) - -router.get('/', - validatorsPagination.pagination, - validatorsSort.videosSort, - sort.setVideosSort, - pagination.setPagination, - listVideos -) -router.put('/:id', - oAuth.authenticate, - reqFiles, - validatorsVideos.videosUpdate, - updateVideoRetryWrapper -) -router.post('/', - oAuth.authenticate, - reqFiles, - validatorsVideos.videosAdd, - addVideoRetryWrapper -) -router.get('/:id', - validatorsVideos.videosGet, - getVideo -) - -router.delete('/:id', - oAuth.authenticate, - validatorsVideos.videosRemove, - removeVideo -) - -router.get('/search/:value', - validatorsVideos.videosSearch, - validatorsPagination.pagination, - validatorsSort.videosSort, - sort.setVideosSort, - pagination.setPagination, - search.setVideosSearch, - searchVideos -) - -// --------------------------------------------------------------------------- - -module.exports = router - -// --------------------------------------------------------------------------- - -function listVideoCategories (req, res, next) { - res.json(constants.VIDEO_CATEGORIES) -} - -function listVideoLicences (req, res, next) { - res.json(constants.VIDEO_LICENCES) -} - -function listVideoLanguages (req, res, next) { - res.json(constants.VIDEO_LANGUAGES) -} - -// Wrapper to video add that retry the function if there is a database error -// We need this because we run the transaction in SERIALIZABLE isolation that can fail -function addVideoRetryWrapper (req, res, next) { - const options = { - arguments: [ req, res, req.files.videofile[0] ], - errorMessage: 'Cannot insert the video with many retries.' - } - - databaseUtils.retryTransactionWrapper(addVideo, options, function (err) { - if (err) return next(err) - - // TODO : include Location of the new video -> 201 - return res.type('json').status(204).end() - }) -} - -function addVideo (req, res, videoFile, finalCallback) { - const videoInfos = req.body - - waterfall([ - - databaseUtils.startSerializableTransaction, - - function findOrCreateAuthor (t, callback) { - const user = res.locals.oauth.token.User - - const name = user.username - // null because it is OUR pod - const podId = null - const userId = user.id - - db.Author.findOrCreateAuthor(name, podId, userId, t, function (err, authorInstance) { - return callback(err, t, authorInstance) - }) - }, - - function findOrCreateTags (t, author, callback) { - const tags = videoInfos.tags - - db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { - return callback(err, t, author, tagInstances) - }) - }, - - function createVideoObject (t, author, tagInstances, callback) { - const videoData = { - name: videoInfos.name, - remoteId: null, - extname: path.extname(videoFile.filename), - category: videoInfos.category, - licence: videoInfos.licence, - language: videoInfos.language, - nsfw: videoInfos.nsfw, - description: videoInfos.description, - duration: videoFile.duration, - authorId: author.id - } - - const video = db.Video.build(videoData) - - return callback(null, t, author, tagInstances, video) - }, - - // Set the videoname the same as the id - function renameVideoFile (t, author, tagInstances, video, callback) { - const videoDir = constants.CONFIG.STORAGE.VIDEOS_DIR - const source = path.join(videoDir, videoFile.filename) - const destination = path.join(videoDir, video.getVideoFilename()) - - fs.rename(source, destination, function (err) { - if (err) return callback(err) - - // This is important in case if there is another attempt - videoFile.filename = video.getVideoFilename() - return callback(null, t, author, tagInstances, video) - }) - }, - - function insertVideoIntoDB (t, author, tagInstances, video, callback) { - const options = { transaction: t } - - // Add tags association - video.save(options).asCallback(function (err, videoCreated) { - if (err) return callback(err) - - // Do not forget to add Author informations to the created video - videoCreated.Author = author - - return callback(err, t, tagInstances, videoCreated) - }) - }, - - function associateTagsToVideo (t, tagInstances, video, callback) { - const options = { transaction: t } - - video.setTags(tagInstances, options).asCallback(function (err) { - video.Tags = tagInstances - - return callback(err, t, video) - }) - }, - - function sendToFriends (t, video, callback) { - // Let transcoding job send the video to friends because the videofile extension might change - if (constants.CONFIG.TRANSCODING.ENABLED === true) return callback(null, t) - - video.toAddRemoteJSON(function (err, remoteVideo) { - if (err) return callback(err) - - // Now we'll add the video's meta data to our friends - friends.addVideoToFriends(remoteVideo, t, function (err) { - return callback(err, t) - }) - }) - }, - - databaseUtils.commitTransaction - - ], function andFinally (err, t) { - if (err) { - // This is just a debug because we will retry the insert - logger.debug('Cannot insert the video.', { error: err }) - return databaseUtils.rollbackTransaction(err, t, finalCallback) - } - - logger.info('Video with name %s created.', videoInfos.name) - return finalCallback(null) - }) -} - -function updateVideoRetryWrapper (req, res, next) { - const options = { - arguments: [ req, res ], - errorMessage: 'Cannot update the video with many retries.' - } - - databaseUtils.retryTransactionWrapper(updateVideo, options, function (err) { - if (err) return next(err) - - // TODO : include Location of the new video -> 201 - return res.type('json').status(204).end() - }) -} - -function updateVideo (req, res, finalCallback) { - const videoInstance = res.locals.video - const videoFieldsSave = videoInstance.toJSON() - const videoInfosToUpdate = req.body - - waterfall([ - - databaseUtils.startSerializableTransaction, - - function findOrCreateTags (t, callback) { - if (videoInfosToUpdate.tags) { - db.Tag.findOrCreateTags(videoInfosToUpdate.tags, t, function (err, tagInstances) { - return callback(err, t, tagInstances) - }) - } else { - return callback(null, t, null) - } - }, - - function updateVideoIntoDB (t, tagInstances, callback) { - const options = { - transaction: t - } - - if (videoInfosToUpdate.name !== undefined) videoInstance.set('name', videoInfosToUpdate.name) - if (videoInfosToUpdate.category !== undefined) videoInstance.set('category', videoInfosToUpdate.category) - if (videoInfosToUpdate.licence !== undefined) videoInstance.set('licence', videoInfosToUpdate.licence) - if (videoInfosToUpdate.language !== undefined) videoInstance.set('language', videoInfosToUpdate.language) - if (videoInfosToUpdate.nsfw !== undefined) videoInstance.set('nsfw', videoInfosToUpdate.nsfw) - if (videoInfosToUpdate.description !== undefined) videoInstance.set('description', videoInfosToUpdate.description) - - videoInstance.save(options).asCallback(function (err) { - return callback(err, t, tagInstances) - }) - }, - - function associateTagsToVideo (t, tagInstances, callback) { - if (tagInstances) { - const options = { transaction: t } - - videoInstance.setTags(tagInstances, options).asCallback(function (err) { - videoInstance.Tags = tagInstances - - return callback(err, t) - }) - } else { - return callback(null, t) - } - }, - - function sendToFriends (t, callback) { - const json = videoInstance.toUpdateRemoteJSON() - - // Now we'll update the video's meta data to our friends - friends.updateVideoToFriends(json, t, function (err) { - return callback(err, t) - }) - }, - - databaseUtils.commitTransaction - - ], function andFinally (err, t) { - if (err) { - logger.debug('Cannot update the video.', { error: err }) - - // Force fields we want to update - // If the transaction is retried, sequelize will think the object has not changed - // So it will skip the SQL request, even if the last one was ROLLBACKed! - Object.keys(videoFieldsSave).forEach(function (key) { - const value = videoFieldsSave[key] - videoInstance.set(key, value) - }) - - return databaseUtils.rollbackTransaction(err, t, finalCallback) - } - - logger.info('Video with name %s updated.', videoInfosToUpdate.name) - return finalCallback(null) - }) -} - -function getVideo (req, res, next) { - const videoInstance = res.locals.video - - if (videoInstance.isOwned()) { - // The increment is done directly in the database, not using the instance value - videoInstance.increment('views').asCallback(function (err) { - if (err) { - logger.error('Cannot add view to video %d.', videoInstance.id) - return - } - - // FIXME: make a real view system - // For example, only add a view when a user watch a video during 30s etc - const qaduParams = { - videoId: videoInstance.id, - type: constants.REQUEST_VIDEO_QADU_TYPES.VIEWS - } - friends.quickAndDirtyUpdateVideoToFriends(qaduParams) - }) - } else { - // Just send the event to our friends - const eventParams = { - videoId: videoInstance.id, - type: constants.REQUEST_VIDEO_EVENT_TYPES.VIEWS - } - friends.addEventToRemoteVideo(eventParams) - } - - // Do not wait the view system - res.json(videoInstance.toFormatedJSON()) -} - -function listVideos (req, res, next) { - db.Video.listForApi(req.query.start, req.query.count, req.query.sort, function (err, videosList, videosTotal) { - if (err) return next(err) - - res.json(utils.getFormatedObjects(videosList, videosTotal)) - }) -} - -function removeVideo (req, res, next) { - const videoInstance = res.locals.video - - videoInstance.destroy().asCallback(function (err) { - if (err) { - logger.error('Errors when removed the video.', { error: err }) - return next(err) - } - - return res.type('json').status(204).end() - }) -} - -function searchVideos (req, res, next) { - db.Video.searchAndPopulateAuthorAndPodAndTags( - req.params.value, req.query.field, req.query.start, req.query.count, req.query.sort, - function (err, videosList, videosTotal) { - if (err) return next(err) - - res.json(utils.getFormatedObjects(videosList, videosTotal)) - } - ) -} diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts new file mode 100644 index 000000000..5fbf03676 --- /dev/null +++ b/server/controllers/api/videos/index.ts @@ -0,0 +1,426 @@ +import express = require('express') +import fs = require('fs') +import multer = require('multer') +import path = require('path') +import { waterfall } from 'async' + +const db = require('../../../initializers/database') +import { + CONFIG, + REQUEST_VIDEO_QADU_TYPES, + REQUEST_VIDEO_EVENT_TYPES, + VIDEO_CATEGORIES, + VIDEO_LICENCES, + VIDEO_LANGUAGES +} from '../../../initializers' +import { + addEventToRemoteVideo, + quickAndDirtyUpdateVideoToFriends, + addVideoToFriends, + updateVideoToFriends +} from '../../../lib' +import { + authenticate, + paginationValidator, + videosSortValidator, + setVideosSort, + setPagination, + setVideosSearch, + videosUpdateValidator, + videosSearchValidator, + videosAddValidator, + videosGetValidator, + videosRemoveValidator +} from '../../../middlewares' +import { + logger, + commitTransaction, + retryTransactionWrapper, + rollbackTransaction, + startSerializableTransaction, + generateRandomString, + getFormatedObjects +} from '../../../helpers' + +import { abuseVideoRouter } from './abuse' +import { blacklistRouter } from './blacklist' +import { rateVideoRouter } from './rate' + +const videosRouter = express.Router() + +// multer configuration +const storage = multer.diskStorage({ + destination: function (req, file, cb) { + cb(null, CONFIG.STORAGE.VIDEOS_DIR) + }, + + filename: function (req, file, cb) { + let extension = '' + if (file.mimetype === 'video/webm') extension = 'webm' + else if (file.mimetype === 'video/mp4') extension = 'mp4' + else if (file.mimetype === 'video/ogg') extension = 'ogv' + generateRandomString(16, function (err, randomString) { + const fieldname = err ? undefined : randomString + cb(null, fieldname + '.' + extension) + }) + } +}) + +const reqFiles = multer({ storage: storage }).fields([{ name: 'videofile', maxCount: 1 }]) + +videosRouter.use('/', abuseVideoRouter) +videosRouter.use('/', blacklistRouter) +videosRouter.use('/', rateVideoRouter) + +videosRouter.get('/categories', listVideoCategories) +videosRouter.get('/licences', listVideoLicences) +videosRouter.get('/languages', listVideoLanguages) + +videosRouter.get('/', + paginationValidator, + videosSortValidator, + setVideosSort, + setPagination, + listVideos +) +videosRouter.put('/:id', + authenticate, + reqFiles, + videosUpdateValidator, + updateVideoRetryWrapper +) +videosRouter.post('/', + authenticate, + reqFiles, + videosAddValidator, + addVideoRetryWrapper +) +videosRouter.get('/:id', + videosGetValidator, + getVideo +) + +videosRouter.delete('/:id', + authenticate, + videosRemoveValidator, + removeVideo +) + +videosRouter.get('/search/:value', + videosSearchValidator, + paginationValidator, + videosSortValidator, + setVideosSort, + setPagination, + setVideosSearch, + searchVideos +) + +// --------------------------------------------------------------------------- + +export { + videosRouter +} + +// --------------------------------------------------------------------------- + +function listVideoCategories (req, res, next) { + res.json(VIDEO_CATEGORIES) +} + +function listVideoLicences (req, res, next) { + res.json(VIDEO_LICENCES) +} + +function listVideoLanguages (req, res, next) { + res.json(VIDEO_LANGUAGES) +} + +// Wrapper to video add that retry the function if there is a database error +// We need this because we run the transaction in SERIALIZABLE isolation that can fail +function addVideoRetryWrapper (req, res, next) { + const options = { + arguments: [ req, res, req.files.videofile[0] ], + errorMessage: 'Cannot insert the video with many retries.' + } + + retryTransactionWrapper(addVideo, options, function (err) { + if (err) return next(err) + + // TODO : include Location of the new video -> 201 + return res.type('json').status(204).end() + }) +} + +function addVideo (req, res, videoFile, finalCallback) { + const videoInfos = req.body + + waterfall([ + + startSerializableTransaction, + + function findOrCreateAuthor (t, callback) { + const user = res.locals.oauth.token.User + + const name = user.username + // null because it is OUR pod + const podId = null + const userId = user.id + + db.Author.findOrCreateAuthor(name, podId, userId, t, function (err, authorInstance) { + return callback(err, t, authorInstance) + }) + }, + + function findOrCreateTags (t, author, callback) { + const tags = videoInfos.tags + + db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { + return callback(err, t, author, tagInstances) + }) + }, + + function createVideoObject (t, author, tagInstances, callback) { + const videoData = { + name: videoInfos.name, + remoteId: null, + extname: path.extname(videoFile.filename), + category: videoInfos.category, + licence: videoInfos.licence, + language: videoInfos.language, + nsfw: videoInfos.nsfw, + description: videoInfos.description, + duration: videoFile.duration, + authorId: author.id + } + + const video = db.Video.build(videoData) + + return callback(null, t, author, tagInstances, video) + }, + + // Set the videoname the same as the id + function renameVideoFile (t, author, tagInstances, video, callback) { + const videoDir = CONFIG.STORAGE.VIDEOS_DIR + const source = path.join(videoDir, videoFile.filename) + const destination = path.join(videoDir, video.getVideoFilename()) + + fs.rename(source, destination, function (err) { + if (err) return callback(err) + + // This is important in case if there is another attempt + videoFile.filename = video.getVideoFilename() + return callback(null, t, author, tagInstances, video) + }) + }, + + function insertVideoIntoDB (t, author, tagInstances, video, callback) { + const options = { transaction: t } + + // Add tags association + video.save(options).asCallback(function (err, videoCreated) { + if (err) return callback(err) + + // Do not forget to add Author informations to the created video + videoCreated.Author = author + + return callback(err, t, tagInstances, videoCreated) + }) + }, + + function associateTagsToVideo (t, tagInstances, video, callback) { + const options = { transaction: t } + + video.setTags(tagInstances, options).asCallback(function (err) { + video.Tags = tagInstances + + return callback(err, t, video) + }) + }, + + function sendToFriends (t, video, callback) { + // Let transcoding job send the video to friends because the videofile extension might change + if (CONFIG.TRANSCODING.ENABLED === true) return callback(null, t) + + video.toAddRemoteJSON(function (err, remoteVideo) { + if (err) return callback(err) + + // Now we'll add the video's meta data to our friends + addVideoToFriends(remoteVideo, t, function (err) { + return callback(err, t) + }) + }) + }, + + commitTransaction + + ], function andFinally (err, t) { + if (err) { + // This is just a debug because we will retry the insert + logger.debug('Cannot insert the video.', { error: err }) + return rollbackTransaction(err, t, finalCallback) + } + + logger.info('Video with name %s created.', videoInfos.name) + return finalCallback(null) + }) +} + +function updateVideoRetryWrapper (req, res, next) { + const options = { + arguments: [ req, res ], + errorMessage: 'Cannot update the video with many retries.' + } + + retryTransactionWrapper(updateVideo, options, function (err) { + if (err) return next(err) + + // TODO : include Location of the new video -> 201 + return res.type('json').status(204).end() + }) +} + +function updateVideo (req, res, finalCallback) { + const videoInstance = res.locals.video + const videoFieldsSave = videoInstance.toJSON() + const videoInfosToUpdate = req.body + + waterfall([ + + startSerializableTransaction, + + function findOrCreateTags (t, callback) { + if (videoInfosToUpdate.tags) { + db.Tag.findOrCreateTags(videoInfosToUpdate.tags, t, function (err, tagInstances) { + return callback(err, t, tagInstances) + }) + } else { + return callback(null, t, null) + } + }, + + function updateVideoIntoDB (t, tagInstances, callback) { + const options = { + transaction: t + } + + if (videoInfosToUpdate.name !== undefined) videoInstance.set('name', videoInfosToUpdate.name) + if (videoInfosToUpdate.category !== undefined) videoInstance.set('category', videoInfosToUpdate.category) + if (videoInfosToUpdate.licence !== undefined) videoInstance.set('licence', videoInfosToUpdate.licence) + if (videoInfosToUpdate.language !== undefined) videoInstance.set('language', videoInfosToUpdate.language) + if (videoInfosToUpdate.nsfw !== undefined) videoInstance.set('nsfw', videoInfosToUpdate.nsfw) + if (videoInfosToUpdate.description !== undefined) videoInstance.set('description', videoInfosToUpdate.description) + + videoInstance.save(options).asCallback(function (err) { + return callback(err, t, tagInstances) + }) + }, + + function associateTagsToVideo (t, tagInstances, callback) { + if (tagInstances) { + const options = { transaction: t } + + videoInstance.setTags(tagInstances, options).asCallback(function (err) { + videoInstance.Tags = tagInstances + + return callback(err, t) + }) + } else { + return callback(null, t) + } + }, + + function sendToFriends (t, callback) { + const json = videoInstance.toUpdateRemoteJSON() + + // Now we'll update the video's meta data to our friends + updateVideoToFriends(json, t, function (err) { + return callback(err, t) + }) + }, + + commitTransaction + + ], function andFinally (err, t) { + if (err) { + logger.debug('Cannot update the video.', { error: err }) + + // Force fields we want to update + // If the transaction is retried, sequelize will think the object has not changed + // So it will skip the SQL request, even if the last one was ROLLBACKed! + Object.keys(videoFieldsSave).forEach(function (key) { + const value = videoFieldsSave[key] + videoInstance.set(key, value) + }) + + return rollbackTransaction(err, t, finalCallback) + } + + logger.info('Video with name %s updated.', videoInfosToUpdate.name) + return finalCallback(null) + }) +} + +function getVideo (req, res, next) { + const videoInstance = res.locals.video + + if (videoInstance.isOwned()) { + // The increment is done directly in the database, not using the instance value + videoInstance.increment('views').asCallback(function (err) { + if (err) { + logger.error('Cannot add view to video %d.', videoInstance.id) + return + } + + // FIXME: make a real view system + // For example, only add a view when a user watch a video during 30s etc + const qaduParams = { + videoId: videoInstance.id, + type: REQUEST_VIDEO_QADU_TYPES.VIEWS + } + quickAndDirtyUpdateVideoToFriends(qaduParams) + }) + } else { + // Just send the event to our friends + const eventParams = { + videoId: videoInstance.id, + type: REQUEST_VIDEO_EVENT_TYPES.VIEWS + } + addEventToRemoteVideo(eventParams) + } + + // Do not wait the view system + res.json(videoInstance.toFormatedJSON()) +} + +function listVideos (req, res, next) { + db.Video.listForApi(req.query.start, req.query.count, req.query.sort, function (err, videosList, videosTotal) { + if (err) return next(err) + + res.json(getFormatedObjects(videosList, videosTotal)) + }) +} + +function removeVideo (req, res, next) { + const videoInstance = res.locals.video + + videoInstance.destroy().asCallback(function (err) { + if (err) { + logger.error('Errors when removed the video.', { error: err }) + return next(err) + } + + return res.type('json').status(204).end() + }) +} + +function searchVideos (req, res, next) { + db.Video.searchAndPopulateAuthorAndPodAndTags( + req.params.value, req.query.field, req.query.start, req.query.count, req.query.sort, + function (err, videosList, videosTotal) { + if (err) return next(err) + + res.json(getFormatedObjects(videosList, videosTotal)) + } + ) +} diff --git a/server/controllers/api/videos/rate.js b/server/controllers/api/videos/rate.js deleted file mode 100644 index df8a69a1d..000000000 --- a/server/controllers/api/videos/rate.js +++ /dev/null @@ -1,169 +0,0 @@ -'use strict' - -const express = require('express') -const waterfall = require('async/waterfall') - -const constants = require('../../../initializers/constants') -const db = require('../../../initializers/database') -const logger = require('../../../helpers/logger') -const friends = require('../../../lib/friends') -const middlewares = require('../../../middlewares') -const oAuth = middlewares.oauth -const validators = middlewares.validators -const validatorsVideos = validators.videos -const databaseUtils = require('../../../helpers/database-utils') - -const router = express.Router() - -router.put('/:id/rate', - oAuth.authenticate, - validatorsVideos.videoRate, - rateVideoRetryWrapper -) - -// --------------------------------------------------------------------------- - -module.exports = router - -// --------------------------------------------------------------------------- - -function rateVideoRetryWrapper (req, res, next) { - const options = { - arguments: [ req, res ], - errorMessage: 'Cannot update the user video rate.' - } - - databaseUtils.retryTransactionWrapper(rateVideo, options, function (err) { - if (err) return next(err) - - return res.type('json').status(204).end() - }) -} - -function rateVideo (req, res, finalCallback) { - const rateType = req.body.rating - const videoInstance = res.locals.video - const userInstance = res.locals.oauth.token.User - - waterfall([ - databaseUtils.startSerializableTransaction, - - function findPreviousRate (t, callback) { - db.UserVideoRate.load(userInstance.id, videoInstance.id, t, function (err, previousRate) { - return callback(err, t, previousRate) - }) - }, - - function insertUserRateIntoDB (t, previousRate, callback) { - const options = { transaction: t } - - let likesToIncrement = 0 - let dislikesToIncrement = 0 - - if (rateType === constants.VIDEO_RATE_TYPES.LIKE) likesToIncrement++ - else if (rateType === constants.VIDEO_RATE_TYPES.DISLIKE) dislikesToIncrement++ - - // There was a previous rate, update it - if (previousRate) { - // We will remove the previous rate, so we will need to remove it from the video attribute - if (previousRate.type === constants.VIDEO_RATE_TYPES.LIKE) likesToIncrement-- - else if (previousRate.type === constants.VIDEO_RATE_TYPES.DISLIKE) dislikesToIncrement-- - - previousRate.type = rateType - - previousRate.save(options).asCallback(function (err) { - return callback(err, t, likesToIncrement, dislikesToIncrement) - }) - } else { // There was not a previous rate, insert a new one - const query = { - userId: userInstance.id, - videoId: videoInstance.id, - type: rateType - } - - db.UserVideoRate.create(query, options).asCallback(function (err) { - return callback(err, t, likesToIncrement, dislikesToIncrement) - }) - } - }, - - function updateVideoAttributeDB (t, likesToIncrement, dislikesToIncrement, callback) { - const options = { transaction: t } - const incrementQuery = { - likes: likesToIncrement, - dislikes: dislikesToIncrement - } - - // Even if we do not own the video we increment the attributes - // It is usefull for the user to have a feedback - videoInstance.increment(incrementQuery, options).asCallback(function (err) { - return callback(err, t, likesToIncrement, dislikesToIncrement) - }) - }, - - function sendEventsToFriendsIfNeeded (t, likesToIncrement, dislikesToIncrement, callback) { - // No need for an event type, we own the video - if (videoInstance.isOwned()) return callback(null, t, likesToIncrement, dislikesToIncrement) - - const eventsParams = [] - - if (likesToIncrement !== 0) { - eventsParams.push({ - videoId: videoInstance.id, - type: constants.REQUEST_VIDEO_EVENT_TYPES.LIKES, - count: likesToIncrement - }) - } - - if (dislikesToIncrement !== 0) { - eventsParams.push({ - videoId: videoInstance.id, - type: constants.REQUEST_VIDEO_EVENT_TYPES.DISLIKES, - count: dislikesToIncrement - }) - } - - friends.addEventsToRemoteVideo(eventsParams, t, function (err) { - return callback(err, t, likesToIncrement, dislikesToIncrement) - }) - }, - - function sendQaduToFriendsIfNeeded (t, likesToIncrement, dislikesToIncrement, callback) { - // We do not own the video, there is no need to send a quick and dirty update to friends - // Our rate was already sent by the addEvent function - if (videoInstance.isOwned() === false) return callback(null, t) - - const qadusParams = [] - - if (likesToIncrement !== 0) { - qadusParams.push({ - videoId: videoInstance.id, - type: constants.REQUEST_VIDEO_QADU_TYPES.LIKES - }) - } - - if (dislikesToIncrement !== 0) { - qadusParams.push({ - videoId: videoInstance.id, - type: constants.REQUEST_VIDEO_QADU_TYPES.DISLIKES - }) - } - - friends.quickAndDirtyUpdatesVideoToFriends(qadusParams, t, function (err) { - return callback(err, t) - }) - }, - - databaseUtils.commitTransaction - - ], function (err, t) { - if (err) { - // This is just a debug because we will retry the insert - logger.debug('Cannot add the user video rate.', { error: err }) - return databaseUtils.rollbackTransaction(err, t, finalCallback) - } - - logger.info('User video rate for video %s of user %s updated.', videoInstance.name, userInstance.username) - return finalCallback(null) - }) -} diff --git a/server/controllers/api/videos/rate.ts b/server/controllers/api/videos/rate.ts new file mode 100644 index 000000000..21053792a --- /dev/null +++ b/server/controllers/api/videos/rate.ts @@ -0,0 +1,181 @@ +import express = require('express') +import { waterfall } from 'async' + +const db = require('../../../initializers/database') +import { + logger, + retryTransactionWrapper, + startSerializableTransaction, + commitTransaction, + rollbackTransaction +} from '../../../helpers' +import { + VIDEO_RATE_TYPES, + REQUEST_VIDEO_EVENT_TYPES, + REQUEST_VIDEO_QADU_TYPES +} from '../../../initializers' +import { + addEventsToRemoteVideo, + quickAndDirtyUpdatesVideoToFriends +} from '../../../lib' +import { + authenticate, + videoRateValidator +} from '../../../middlewares' + +const rateVideoRouter = express.Router() + +rateVideoRouter.put('/:id/rate', + authenticate, + videoRateValidator, + rateVideoRetryWrapper +) + +// --------------------------------------------------------------------------- + +export { + rateVideoRouter +} + +// --------------------------------------------------------------------------- + +function rateVideoRetryWrapper (req, res, next) { + const options = { + arguments: [ req, res ], + errorMessage: 'Cannot update the user video rate.' + } + + retryTransactionWrapper(rateVideo, options, function (err) { + if (err) return next(err) + + return res.type('json').status(204).end() + }) +} + +function rateVideo (req, res, finalCallback) { + const rateType = req.body.rating + const videoInstance = res.locals.video + const userInstance = res.locals.oauth.token.User + + waterfall([ + startSerializableTransaction, + + function findPreviousRate (t, callback) { + db.UserVideoRate.load(userInstance.id, videoInstance.id, t, function (err, previousRate) { + return callback(err, t, previousRate) + }) + }, + + function insertUserRateIntoDB (t, previousRate, callback) { + const options = { transaction: t } + + let likesToIncrement = 0 + let dislikesToIncrement = 0 + + if (rateType === VIDEO_RATE_TYPES.LIKE) likesToIncrement++ + else if (rateType === VIDEO_RATE_TYPES.DISLIKE) dislikesToIncrement++ + + // There was a previous rate, update it + if (previousRate) { + // We will remove the previous rate, so we will need to remove it from the video attribute + if (previousRate.type === VIDEO_RATE_TYPES.LIKE) likesToIncrement-- + else if (previousRate.type === VIDEO_RATE_TYPES.DISLIKE) dislikesToIncrement-- + + previousRate.type = rateType + + previousRate.save(options).asCallback(function (err) { + return callback(err, t, likesToIncrement, dislikesToIncrement) + }) + } else { // There was not a previous rate, insert a new one + const query = { + userId: userInstance.id, + videoId: videoInstance.id, + type: rateType + } + + db.UserVideoRate.create(query, options).asCallback(function (err) { + return callback(err, t, likesToIncrement, dislikesToIncrement) + }) + } + }, + + function updateVideoAttributeDB (t, likesToIncrement, dislikesToIncrement, callback) { + const options = { transaction: t } + const incrementQuery = { + likes: likesToIncrement, + dislikes: dislikesToIncrement + } + + // Even if we do not own the video we increment the attributes + // It is usefull for the user to have a feedback + videoInstance.increment(incrementQuery, options).asCallback(function (err) { + return callback(err, t, likesToIncrement, dislikesToIncrement) + }) + }, + + function sendEventsToFriendsIfNeeded (t, likesToIncrement, dislikesToIncrement, callback) { + // No need for an event type, we own the video + if (videoInstance.isOwned()) return callback(null, t, likesToIncrement, dislikesToIncrement) + + const eventsParams = [] + + if (likesToIncrement !== 0) { + eventsParams.push({ + videoId: videoInstance.id, + type: REQUEST_VIDEO_EVENT_TYPES.LIKES, + count: likesToIncrement + }) + } + + if (dislikesToIncrement !== 0) { + eventsParams.push({ + videoId: videoInstance.id, + type: REQUEST_VIDEO_EVENT_TYPES.DISLIKES, + count: dislikesToIncrement + }) + } + + addEventsToRemoteVideo(eventsParams, t, function (err) { + return callback(err, t, likesToIncrement, dislikesToIncrement) + }) + }, + + function sendQaduToFriendsIfNeeded (t, likesToIncrement, dislikesToIncrement, callback) { + // We do not own the video, there is no need to send a quick and dirty update to friends + // Our rate was already sent by the addEvent function + if (videoInstance.isOwned() === false) return callback(null, t) + + const qadusParams = [] + + if (likesToIncrement !== 0) { + qadusParams.push({ + videoId: videoInstance.id, + type: REQUEST_VIDEO_QADU_TYPES.LIKES + }) + } + + if (dislikesToIncrement !== 0) { + qadusParams.push({ + videoId: videoInstance.id, + type: REQUEST_VIDEO_QADU_TYPES.DISLIKES + }) + } + + quickAndDirtyUpdatesVideoToFriends(qadusParams, t, function (err) { + return callback(err, t) + }) + }, + + commitTransaction + + ], function (err, t) { + if (err) { + // This is just a debug because we will retry the insert + logger.debug('Cannot add the user video rate.', { error: err }) + return rollbackTransaction(err, t, finalCallback) + } + + logger.info('User video rate for video %s of user %s updated.', videoInstance.name, userInstance.username) + return finalCallback(null) + }) +} diff --git a/server/controllers/client.js b/server/controllers/client.js deleted file mode 100644 index 83243a4f7..000000000 --- a/server/controllers/client.js +++ /dev/null @@ -1,110 +0,0 @@ -'use strict' - -const parallel = require('async/parallel') -const express = require('express') -const fs = require('fs') -const path = require('path') -const validator = require('express-validator').validator - -const constants = require('../initializers/constants') -const db = require('../initializers/database') - -const router = express.Router() - -const opengraphComment = '' -const distPath = path.join(__dirname, '..', '..', 'client/dist') -const embedPath = path.join(distPath, 'standalone/videos/embed.html') -const indexPath = path.join(distPath, 'index.html') - -// Special route that add OpenGraph tags -// Do not use a template engine for a so little thing -router.use('/videos/watch/:id', generateWatchHtmlPage) - -router.use('/videos/embed', function (req, res, next) { - res.sendFile(embedPath) -}) - -// Static HTML/CSS/JS client files -router.use('/client', express.static(distPath, { maxAge: constants.STATIC_MAX_AGE })) - -// 404 for static files not found -router.use('/client/*', function (req, res, next) { - res.sendStatus(404) -}) - -// --------------------------------------------------------------------------- - -module.exports = router - -// --------------------------------------------------------------------------- - -function addOpenGraphTags (htmlStringPage, video) { - let basePreviewUrlHttp - - if (video.isOwned()) { - basePreviewUrlHttp = constants.CONFIG.WEBSERVER.URL - } else { - basePreviewUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + video.Author.Pod.host - } - - // We fetch the remote preview (bigger than the thumbnail) - // This should not overhead the remote server since social websites put in a cache the OpenGraph tags - // We can't use the thumbnail because these social websites want bigger images (> 200x200 for Facebook for example) - const previewUrl = basePreviewUrlHttp + constants.STATIC_PATHS.PREVIEWS + video.getPreviewName() - const videoUrl = constants.CONFIG.WEBSERVER.URL + '/videos/watch/' + video.id - - const metaTags = { - 'og:type': 'video', - 'og:title': video.name, - 'og:image': previewUrl, - 'og:url': videoUrl, - 'og:description': video.description, - - 'name': video.name, - 'description': video.description, - 'image': previewUrl, - - 'twitter:card': 'summary_large_image', - 'twitter:site': '@Chocobozzz', - 'twitter:title': video.name, - 'twitter:description': video.description, - 'twitter:image': previewUrl - } - - let tagsString = '' - Object.keys(metaTags).forEach(function (tagName) { - const tagValue = metaTags[tagName] - - tagsString += '' - }) - - return htmlStringPage.replace(opengraphComment, tagsString) -} - -function generateWatchHtmlPage (req, res, next) { - const videoId = req.params.id - - // Let Angular application handle errors - if (!validator.isUUID(videoId, 4)) return res.sendFile(indexPath) - - parallel({ - file: function (callback) { - fs.readFile(indexPath, callback) - }, - - video: function (callback) { - db.Video.loadAndPopulateAuthorAndPodAndTags(videoId, callback) - } - }, function (err, results) { - if (err) return next(err) - - const html = results.file.toString() - const video = results.video - - // Let Angular application handle errors - if (!video) return res.sendFile(indexPath) - - const htmlStringPageWithTags = addOpenGraphTags(html, video) - res.set('Content-Type', 'text/html; charset=UTF-8').send(htmlStringPageWithTags) - }) -} diff --git a/server/controllers/client.ts b/server/controllers/client.ts new file mode 100644 index 000000000..aaa04889a --- /dev/null +++ b/server/controllers/client.ts @@ -0,0 +1,118 @@ +import { parallel } from 'async' +import express = require('express') +import fs = require('fs') +import { join } from 'path' +import expressValidator = require('express-validator') +// TODO: use .validator when express-validator typing will have validator field +const validator = expressValidator['validator'] + +const db = require('../initializers/database') +import { + CONFIG, + REMOTE_SCHEME, + STATIC_PATHS, + STATIC_MAX_AGE +} from '../initializers' + +const clientsRouter = express.Router() + +// TODO: move to constants +const opengraphComment = '' +const distPath = join(__dirname, '..', '..', 'client/dist') +const embedPath = join(distPath, 'standalone/videos/embed.html') +const indexPath = join(distPath, 'index.html') + +// Special route that add OpenGraph tags +// Do not use a template engine for a so little thing +clientsRouter.use('/videos/watch/:id', generateWatchHtmlPage) + +clientsRouter.use('/videos/embed', function (req, res, next) { + res.sendFile(embedPath) +}) + +// Static HTML/CSS/JS client files +clientsRouter.use('/client', express.static(distPath, { maxAge: STATIC_MAX_AGE })) + +// 404 for static files not found +clientsRouter.use('/client/*', function (req, res, next) { + res.sendStatus(404) +}) + +// --------------------------------------------------------------------------- + +export { + clientsRouter +} + +// --------------------------------------------------------------------------- + +function addOpenGraphTags (htmlStringPage, video) { + let basePreviewUrlHttp + + if (video.isOwned()) { + basePreviewUrlHttp = CONFIG.WEBSERVER.URL + } else { + basePreviewUrlHttp = REMOTE_SCHEME.HTTP + '://' + video.Author.Pod.host + } + + // We fetch the remote preview (bigger than the thumbnail) + // This should not overhead the remote server since social websites put in a cache the OpenGraph tags + // We can't use the thumbnail because these social websites want bigger images (> 200x200 for Facebook for example) + const previewUrl = basePreviewUrlHttp + STATIC_PATHS.PREVIEWS + video.getPreviewName() + const videoUrl = CONFIG.WEBSERVER.URL + '/videos/watch/' + video.id + + const metaTags = { + 'og:type': 'video', + 'og:title': video.name, + 'og:image': previewUrl, + 'og:url': videoUrl, + 'og:description': video.description, + + 'name': video.name, + 'description': video.description, + 'image': previewUrl, + + 'twitter:card': 'summary_large_image', + 'twitter:site': '@Chocobozzz', + 'twitter:title': video.name, + 'twitter:description': video.description, + 'twitter:image': previewUrl + } + + let tagsString = '' + Object.keys(metaTags).forEach(function (tagName) { + const tagValue = metaTags[tagName] + + tagsString += '' + }) + + return htmlStringPage.replace(opengraphComment, tagsString) +} + +function generateWatchHtmlPage (req, res, next) { + const videoId = req.params.id + + // Let Angular application handle errors + if (!validator.isUUID(videoId, 4)) return res.sendFile(indexPath) + + parallel({ + file: function (callback) { + fs.readFile(indexPath, callback) + }, + + video: function (callback) { + db.Video.loadAndPopulateAuthorAndPodAndTags(videoId, callback) + } + }, function (err, result: any) { + if (err) return next(err) + + const html = result.file.toString() + const video = result.video + + // Let Angular application handle errors + if (!video) return res.sendFile(indexPath) + + const htmlStringPageWithTags = addOpenGraphTags(html, video) + res.set('Content-Type', 'text/html; charset=UTF-8').send(htmlStringPageWithTags) + }) +} diff --git a/server/controllers/index.js b/server/controllers/index.js deleted file mode 100644 index c9ca297ef..000000000 --- a/server/controllers/index.js +++ /dev/null @@ -1,11 +0,0 @@ -'use strict' - -const apiController = require('./api/') -const clientController = require('./client') -const staticController = require('./static') - -module.exports = { - api: apiController, - client: clientController, - static: staticController -} diff --git a/server/controllers/index.ts b/server/controllers/index.ts new file mode 100644 index 000000000..bb56fd7cf --- /dev/null +++ b/server/controllers/index.ts @@ -0,0 +1,3 @@ +export * from './static'; +export * from './client'; +export * from './api'; diff --git a/server/controllers/static.js b/server/controllers/static.js deleted file mode 100644 index 810b752af..000000000 --- a/server/controllers/static.js +++ /dev/null @@ -1,45 +0,0 @@ -'use strict' - -const express = require('express') -const cors = require('cors') - -const constants = require('../initializers/constants') - -const router = express.Router() - -/* - Cors is very important to let other pods access torrent and video files -*/ - -const torrentsPhysicalPath = constants.CONFIG.STORAGE.TORRENTS_DIR -router.use( - constants.STATIC_PATHS.TORRENTS, - cors(), - express.static(torrentsPhysicalPath, { maxAge: constants.STATIC_MAX_AGE }) -) - -// Videos path for webseeding -const videosPhysicalPath = constants.CONFIG.STORAGE.VIDEOS_DIR -router.use( - constants.STATIC_PATHS.WEBSEED, - cors(), - express.static(videosPhysicalPath, { maxAge: constants.STATIC_MAX_AGE }) -) - -// Thumbnails path for express -const thumbnailsPhysicalPath = constants.CONFIG.STORAGE.THUMBNAILS_DIR -router.use( - constants.STATIC_PATHS.THUMBNAILS, - express.static(thumbnailsPhysicalPath, { maxAge: constants.STATIC_MAX_AGE }) -) - -// Video previews path for express -const previewsPhysicalPath = constants.CONFIG.STORAGE.PREVIEWS_DIR -router.use( - constants.STATIC_PATHS.PREVIEWS, - express.static(previewsPhysicalPath, { maxAge: constants.STATIC_MAX_AGE }) -) - -// --------------------------------------------------------------------------- - -module.exports = router diff --git a/server/controllers/static.ts b/server/controllers/static.ts new file mode 100644 index 000000000..51f75c57e --- /dev/null +++ b/server/controllers/static.ts @@ -0,0 +1,49 @@ +import express = require('express') +import cors = require('cors') + +import { + CONFIG, + STATIC_MAX_AGE, + STATIC_PATHS +} from '../initializers' + +const staticRouter = express.Router() + +/* + Cors is very important to let other pods access torrent and video files +*/ + +const torrentsPhysicalPath = CONFIG.STORAGE.TORRENTS_DIR +staticRouter.use( + STATIC_PATHS.TORRENTS, + cors(), + express.static(torrentsPhysicalPath, { maxAge: STATIC_MAX_AGE }) +) + +// Videos path for webseeding +const videosPhysicalPath = CONFIG.STORAGE.VIDEOS_DIR +staticRouter.use( + STATIC_PATHS.WEBSEED, + cors(), + express.static(videosPhysicalPath, { maxAge: STATIC_MAX_AGE }) +) + +// Thumbnails path for express +const thumbnailsPhysicalPath = CONFIG.STORAGE.THUMBNAILS_DIR +staticRouter.use( + STATIC_PATHS.THUMBNAILS, + express.static(thumbnailsPhysicalPath, { maxAge: STATIC_MAX_AGE }) +) + +// Video previews path for express +const previewsPhysicalPath = CONFIG.STORAGE.PREVIEWS_DIR +staticRouter.use( + STATIC_PATHS.PREVIEWS, + express.static(previewsPhysicalPath, { maxAge: STATIC_MAX_AGE }) +) + +// --------------------------------------------------------------------------- + +export { + staticRouter +} diff --git a/server/helpers/custom-validators/index.js b/server/helpers/custom-validators/index.js deleted file mode 100644 index 9383e0304..000000000 --- a/server/helpers/custom-validators/index.js +++ /dev/null @@ -1,19 +0,0 @@ -'use strict' - -const miscValidators = require('./misc') -const podsValidators = require('./pods') -const remoteValidators = require('./remote') -const usersValidators = require('./users') -const videosValidators = require('./videos') - -const validators = { - misc: miscValidators, - pods: podsValidators, - remote: remoteValidators, - users: usersValidators, - videos: videosValidators -} - -// --------------------------------------------------------------------------- - -module.exports = validators diff --git a/server/helpers/custom-validators/index.ts b/server/helpers/custom-validators/index.ts new file mode 100644 index 000000000..1dcab624a --- /dev/null +++ b/server/helpers/custom-validators/index.ts @@ -0,0 +1,6 @@ +export * from './remote' +export * from './misc' +export * from './pods' +export * from './pods' +export * from './users' +export * from './videos' diff --git a/server/helpers/custom-validators/misc.js b/server/helpers/custom-validators/misc.js deleted file mode 100644 index 052726241..000000000 --- a/server/helpers/custom-validators/misc.js +++ /dev/null @@ -1,18 +0,0 @@ -'use strict' - -const miscValidators = { - exists, - isArray -} - -function exists (value) { - return value !== undefined && value !== null -} - -function isArray (value) { - return Array.isArray(value) -} - -// --------------------------------------------------------------------------- - -module.exports = miscValidators diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts new file mode 100644 index 000000000..83f50a7fe --- /dev/null +++ b/server/helpers/custom-validators/misc.ts @@ -0,0 +1,14 @@ +function exists (value) { + return value !== undefined && value !== null +} + +function isArray (value) { + return Array.isArray(value) +} + +// --------------------------------------------------------------------------- + +export { + exists, + isArray +} diff --git a/server/helpers/custom-validators/pods.js b/server/helpers/custom-validators/pods.js deleted file mode 100644 index 8bb3733ff..000000000 --- a/server/helpers/custom-validators/pods.js +++ /dev/null @@ -1,26 +0,0 @@ -'use strict' - -const validator = require('express-validator').validator - -const miscValidators = require('./misc') - -const podsValidators = { - isEachUniqueHostValid, - isHostValid -} - -function isHostValid (host) { - return validator.isURL(host) && host.split('://').length === 1 -} - -function isEachUniqueHostValid (hosts) { - return miscValidators.isArray(hosts) && - hosts.length !== 0 && - hosts.every(function (host) { - return isHostValid(host) && hosts.indexOf(host) === hosts.lastIndexOf(host) - }) -} - -// --------------------------------------------------------------------------- - -module.exports = podsValidators diff --git a/server/helpers/custom-validators/pods.ts b/server/helpers/custom-validators/pods.ts new file mode 100644 index 000000000..e4c827feb --- /dev/null +++ b/server/helpers/custom-validators/pods.ts @@ -0,0 +1,24 @@ +import expressValidator = require('express-validator') +// TODO: use .validator when express-validator typing will have validator field +const validator = expressValidator['validator'] + +import { isArray } from './misc' + +function isHostValid (host) { + return validator.isURL(host) && host.split('://').length === 1 +} + +function isEachUniqueHostValid (hosts) { + return isArray(hosts) && + hosts.length !== 0 && + hosts.every(function (host) { + return isHostValid(host) && hosts.indexOf(host) === hosts.lastIndexOf(host) + }) +} + +// --------------------------------------------------------------------------- + +export { + isEachUniqueHostValid, + isHostValid +} diff --git a/server/helpers/custom-validators/remote/index.js b/server/helpers/custom-validators/remote/index.js deleted file mode 100644 index 1939a95f4..000000000 --- a/server/helpers/custom-validators/remote/index.js +++ /dev/null @@ -1,11 +0,0 @@ -'use strict' - -const remoteVideosValidators = require('./videos') - -const validators = { - videos: remoteVideosValidators -} - -// --------------------------------------------------------------------------- - -module.exports = validators diff --git a/server/helpers/custom-validators/remote/index.ts b/server/helpers/custom-validators/remote/index.ts new file mode 100644 index 000000000..d6f9a7e77 --- /dev/null +++ b/server/helpers/custom-validators/remote/index.ts @@ -0,0 +1 @@ +export * from './videos'; diff --git a/server/helpers/custom-validators/remote/videos.js b/server/helpers/custom-validators/remote/videos.js deleted file mode 100644 index 24715b4b3..000000000 --- a/server/helpers/custom-validators/remote/videos.js +++ /dev/null @@ -1,118 +0,0 @@ -'use strict' - -const has = require('lodash/has') -const values = require('lodash/values') - -const constants = require('../../../initializers/constants') -const videosValidators = require('../videos') -const miscValidators = require('../misc') - -const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS] - -const remoteVideosValidators = { - isEachRemoteRequestVideosValid, - isEachRemoteRequestVideosQaduValid, - isEachRemoteRequestVideosEventsValid -} - -function isEachRemoteRequestVideosValid (requests) { - return miscValidators.isArray(requests) && - requests.every(function (request) { - const video = request.data - - if (!video) return false - - return ( - isRequestTypeAddValid(request.type) && - isCommonVideoAttributesValid(video) && - videosValidators.isVideoAuthorValid(video.author) && - videosValidators.isVideoThumbnailDataValid(video.thumbnailData) - ) || - ( - isRequestTypeUpdateValid(request.type) && - isCommonVideoAttributesValid(video) - ) || - ( - isRequestTypeRemoveValid(request.type) && - videosValidators.isVideoRemoteIdValid(video.remoteId) - ) || - ( - isRequestTypeReportAbuseValid(request.type) && - videosValidators.isVideoRemoteIdValid(request.data.videoRemoteId) && - videosValidators.isVideoAbuseReasonValid(request.data.reportReason) && - videosValidators.isVideoAbuseReporterUsernameValid(request.data.reporterUsername) - ) - }) -} - -function isEachRemoteRequestVideosQaduValid (requests) { - return miscValidators.isArray(requests) && - requests.every(function (request) { - const video = request.data - - if (!video) return false - - return ( - videosValidators.isVideoRemoteIdValid(video.remoteId) && - (has(video, 'views') === false || videosValidators.isVideoViewsValid) && - (has(video, 'likes') === false || videosValidators.isVideoLikesValid) && - (has(video, 'dislikes') === false || videosValidators.isVideoDislikesValid) - ) - }) -} - -function isEachRemoteRequestVideosEventsValid (requests) { - return miscValidators.isArray(requests) && - requests.every(function (request) { - const eventData = request.data - - if (!eventData) return false - - return ( - videosValidators.isVideoRemoteIdValid(eventData.remoteId) && - values(constants.REQUEST_VIDEO_EVENT_TYPES).indexOf(eventData.eventType) !== -1 && - videosValidators.isVideoEventCountValid(eventData.count) - ) - }) -} - -// --------------------------------------------------------------------------- - -module.exports = remoteVideosValidators - -// --------------------------------------------------------------------------- - -function isCommonVideoAttributesValid (video) { - return videosValidators.isVideoDateValid(video.createdAt) && - videosValidators.isVideoDateValid(video.updatedAt) && - videosValidators.isVideoCategoryValid(video.category) && - videosValidators.isVideoLicenceValid(video.licence) && - videosValidators.isVideoLanguageValid(video.language) && - videosValidators.isVideoNSFWValid(video.nsfw) && - videosValidators.isVideoDescriptionValid(video.description) && - videosValidators.isVideoDurationValid(video.duration) && - videosValidators.isVideoInfoHashValid(video.infoHash) && - videosValidators.isVideoNameValid(video.name) && - videosValidators.isVideoTagsValid(video.tags) && - videosValidators.isVideoRemoteIdValid(video.remoteId) && - videosValidators.isVideoExtnameValid(video.extname) && - videosValidators.isVideoViewsValid(video.views) && - videosValidators.isVideoLikesValid(video.likes) && - videosValidators.isVideoDislikesValid(video.dislikes) -} - -function isRequestTypeAddValid (value) { - return value === ENDPOINT_ACTIONS.ADD -} - -function isRequestTypeUpdateValid (value) { - return value === ENDPOINT_ACTIONS.UPDATE -} - -function isRequestTypeRemoveValid (value) { - return value === ENDPOINT_ACTIONS.REMOVE -} - -function isRequestTypeReportAbuseValid (value) { - return value === ENDPOINT_ACTIONS.REPORT_ABUSE -} diff --git a/server/helpers/custom-validators/remote/videos.ts b/server/helpers/custom-validators/remote/videos.ts new file mode 100644 index 000000000..4b904d011 --- /dev/null +++ b/server/helpers/custom-validators/remote/videos.ts @@ -0,0 +1,138 @@ +import { has, values } from 'lodash' + +import { + REQUEST_ENDPOINTS, + REQUEST_ENDPOINT_ACTIONS, + REQUEST_VIDEO_EVENT_TYPES +} from '../../../initializers' +import { isArray } from '../misc' +import { + isVideoAuthorValid, + isVideoThumbnailDataValid, + isVideoRemoteIdValid, + isVideoAbuseReasonValid, + isVideoAbuseReporterUsernameValid, + isVideoViewsValid, + isVideoLikesValid, + isVideoDislikesValid, + isVideoEventCountValid, + isVideoDateValid, + isVideoCategoryValid, + isVideoLicenceValid, + isVideoLanguageValid, + isVideoNSFWValid, + isVideoDescriptionValid, + isVideoDurationValid, + isVideoInfoHashValid, + isVideoNameValid, + isVideoTagsValid, + isVideoExtnameValid +} from '../videos' + +const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] + +function isEachRemoteRequestVideosValid (requests) { + return isArray(requests) && + requests.every(function (request) { + const video = request.data + + if (!video) return false + + return ( + isRequestTypeAddValid(request.type) && + isCommonVideoAttributesValid(video) && + isVideoAuthorValid(video.author) && + isVideoThumbnailDataValid(video.thumbnailData) + ) || + ( + isRequestTypeUpdateValid(request.type) && + isCommonVideoAttributesValid(video) + ) || + ( + isRequestTypeRemoveValid(request.type) && + isVideoRemoteIdValid(video.remoteId) + ) || + ( + isRequestTypeReportAbuseValid(request.type) && + isVideoRemoteIdValid(request.data.videoRemoteId) && + isVideoAbuseReasonValid(request.data.reportReason) && + isVideoAbuseReporterUsernameValid(request.data.reporterUsername) + ) + }) +} + +function isEachRemoteRequestVideosQaduValid (requests) { + return isArray(requests) && + requests.every(function (request) { + const video = request.data + + if (!video) return false + + return ( + isVideoRemoteIdValid(video.remoteId) && + (has(video, 'views') === false || isVideoViewsValid) && + (has(video, 'likes') === false || isVideoLikesValid) && + (has(video, 'dislikes') === false || isVideoDislikesValid) + ) + }) +} + +function isEachRemoteRequestVideosEventsValid (requests) { + return isArray(requests) && + requests.every(function (request) { + const eventData = request.data + + if (!eventData) return false + + return ( + isVideoRemoteIdValid(eventData.remoteId) && + values(REQUEST_VIDEO_EVENT_TYPES).indexOf(eventData.eventType) !== -1 && + isVideoEventCountValid(eventData.count) + ) + }) +} + +// --------------------------------------------------------------------------- + +export { + isEachRemoteRequestVideosValid, + isEachRemoteRequestVideosQaduValid, + isEachRemoteRequestVideosEventsValid +} + +// --------------------------------------------------------------------------- + +function isCommonVideoAttributesValid (video) { + return isVideoDateValid(video.createdAt) && + isVideoDateValid(video.updatedAt) && + isVideoCategoryValid(video.category) && + isVideoLicenceValid(video.licence) && + isVideoLanguageValid(video.language) && + isVideoNSFWValid(video.nsfw) && + isVideoDescriptionValid(video.description) && + isVideoDurationValid(video.duration) && + isVideoInfoHashValid(video.infoHash) && + isVideoNameValid(video.name) && + isVideoTagsValid(video.tags) && + isVideoRemoteIdValid(video.remoteId) && + isVideoExtnameValid(video.extname) && + isVideoViewsValid(video.views) && + isVideoLikesValid(video.likes) && + isVideoDislikesValid(video.dislikes) +} + +function isRequestTypeAddValid (value) { + return value === ENDPOINT_ACTIONS.ADD +} + +function isRequestTypeUpdateValid (value) { + return value === ENDPOINT_ACTIONS.UPDATE +} + +function isRequestTypeRemoveValid (value) { + return value === ENDPOINT_ACTIONS.REMOVE +} + +function isRequestTypeReportAbuseValid (value) { + return value === ENDPOINT_ACTIONS.REPORT_ABUSE +} diff --git a/server/helpers/custom-validators/users.js b/server/helpers/custom-validators/users.js deleted file mode 100644 index 2fc026e98..000000000 --- a/server/helpers/custom-validators/users.js +++ /dev/null @@ -1,36 +0,0 @@ -'use strict' - -const validator = require('express-validator').validator -const values = require('lodash/values') - -const constants = require('../../initializers/constants') -const USERS_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.USERS - -const usersValidators = { - isUserPasswordValid, - isUserRoleValid, - isUserUsernameValid, - isUserDisplayNSFWValid -} - -function isUserPasswordValid (value) { - return validator.isLength(value, USERS_CONSTRAINTS_FIELDS.PASSWORD) -} - -function isUserRoleValid (value) { - return values(constants.USER_ROLES).indexOf(value) !== -1 -} - -function isUserUsernameValid (value) { - const max = USERS_CONSTRAINTS_FIELDS.USERNAME.max - const min = USERS_CONSTRAINTS_FIELDS.USERNAME.min - return validator.matches(value, new RegExp(`^[a-zA-Z0-9._]{${min},${max}}$`)) -} - -function isUserDisplayNSFWValid (value) { - return validator.isBoolean(value) -} - -// --------------------------------------------------------------------------- - -module.exports = usersValidators diff --git a/server/helpers/custom-validators/users.ts b/server/helpers/custom-validators/users.ts new file mode 100644 index 000000000..8fd2dac4f --- /dev/null +++ b/server/helpers/custom-validators/users.ts @@ -0,0 +1,34 @@ +import { values } from 'lodash' +import expressValidator = require('express-validator') +// TODO: use .validator when express-validator typing will have validator field +const validator = expressValidator['validator'] + +import { CONSTRAINTS_FIELDS, USER_ROLES } from '../../initializers' +const USERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.USERS + +function isUserPasswordValid (value) { + return validator.isLength(value, USERS_CONSTRAINTS_FIELDS.PASSWORD) +} + +function isUserRoleValid (value) { + return values(USER_ROLES).indexOf(value) !== -1 +} + +function isUserUsernameValid (value) { + const max = USERS_CONSTRAINTS_FIELDS.USERNAME.max + const min = USERS_CONSTRAINTS_FIELDS.USERNAME.min + return validator.matches(value, new RegExp(`^[a-zA-Z0-9._]{${min},${max}}$`)) +} + +function isUserDisplayNSFWValid (value) { + return validator.isBoolean(value) +} + +// --------------------------------------------------------------------------- + +export { + isUserPasswordValid, + isUserRoleValid, + isUserUsernameValid, + isUserDisplayNSFWValid +} diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js deleted file mode 100644 index 196731e04..000000000 --- a/server/helpers/custom-validators/videos.js +++ /dev/null @@ -1,148 +0,0 @@ -'use strict' - -const validator = require('express-validator').validator -const values = require('lodash/values') - -const constants = require('../../initializers/constants') -const usersValidators = require('./users') -const miscValidators = require('./misc') -const VIDEOS_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEOS -const VIDEO_ABUSES_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEO_ABUSES -const VIDEO_EVENTS_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEO_EVENTS - -const videosValidators = { - isVideoAuthorValid, - isVideoDateValid, - isVideoCategoryValid, - isVideoLicenceValid, - isVideoLanguageValid, - isVideoNSFWValid, - isVideoDescriptionValid, - isVideoDurationValid, - isVideoInfoHashValid, - isVideoNameValid, - isVideoTagsValid, - isVideoThumbnailValid, - isVideoThumbnailDataValid, - isVideoExtnameValid, - isVideoRemoteIdValid, - isVideoAbuseReasonValid, - isVideoAbuseReporterUsernameValid, - isVideoFile, - isVideoViewsValid, - isVideoLikesValid, - isVideoRatingTypeValid, - isVideoDislikesValid, - isVideoEventCountValid -} - -function isVideoAuthorValid (value) { - return usersValidators.isUserUsernameValid(value) -} - -function isVideoDateValid (value) { - return validator.isDate(value) -} - -function isVideoCategoryValid (value) { - return constants.VIDEO_CATEGORIES[value] !== undefined -} - -function isVideoLicenceValid (value) { - return constants.VIDEO_LICENCES[value] !== undefined -} - -function isVideoLanguageValid (value) { - return value === null || constants.VIDEO_LANGUAGES[value] !== undefined -} - -function isVideoNSFWValid (value) { - return validator.isBoolean(value) -} - -function isVideoDescriptionValid (value) { - return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.DESCRIPTION) -} - -function isVideoDurationValid (value) { - return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION) -} - -function isVideoExtnameValid (value) { - return VIDEOS_CONSTRAINTS_FIELDS.EXTNAME.indexOf(value) !== -1 -} - -function isVideoInfoHashValid (value) { - return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.INFO_HASH) -} - -function isVideoNameValid (value) { - return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.NAME) -} - -function isVideoTagsValid (tags) { - return miscValidators.isArray(tags) && - validator.isInt(tags.length, VIDEOS_CONSTRAINTS_FIELDS.TAGS) && - tags.every(function (tag) { - return validator.isLength(tag, VIDEOS_CONSTRAINTS_FIELDS.TAG) - }) -} - -function isVideoThumbnailValid (value) { - return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.THUMBNAIL) -} - -function isVideoThumbnailDataValid (value) { - return validator.isByteLength(value, VIDEOS_CONSTRAINTS_FIELDS.THUMBNAIL_DATA) -} - -function isVideoRemoteIdValid (value) { - return validator.isUUID(value, 4) -} - -function isVideoAbuseReasonValid (value) { - return validator.isLength(value, VIDEO_ABUSES_CONSTRAINTS_FIELDS.REASON) -} - -function isVideoAbuseReporterUsernameValid (value) { - return usersValidators.isUserUsernameValid(value) -} - -function isVideoViewsValid (value) { - return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.VIEWS) -} - -function isVideoLikesValid (value) { - return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.LIKES) -} - -function isVideoDislikesValid (value) { - return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DISLIKES) -} - -function isVideoEventCountValid (value) { - return validator.isInt(value + '', VIDEO_EVENTS_CONSTRAINTS_FIELDS.COUNT) -} - -function isVideoRatingTypeValid (value) { - return values(constants.VIDEO_RATE_TYPES).indexOf(value) !== -1 -} - -function isVideoFile (value, files) { - // Should have files - if (!files) return false - - // Should have videofile file - const videofile = files.videofile - if (!videofile || videofile.length === 0) return false - - // The file should exist - const file = videofile[0] - if (!file || !file.originalname) return false - - return new RegExp('^video/(webm|mp4|ogg)$', 'i').test(file.mimetype) -} - -// --------------------------------------------------------------------------- - -module.exports = videosValidators diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts new file mode 100644 index 000000000..2b2370be4 --- /dev/null +++ b/server/helpers/custom-validators/videos.ts @@ -0,0 +1,153 @@ +import { values } from 'lodash' +import expressValidator = require('express-validator') +// TODO: use .validator when express-validator typing will have validator field +const validator = expressValidator['validator'] + +import { + CONSTRAINTS_FIELDS, + VIDEO_CATEGORIES, + VIDEO_LICENCES, + VIDEO_LANGUAGES, + VIDEO_RATE_TYPES +} from '../../initializers' +import { isUserUsernameValid } from './users' +import { isArray } from './misc' + +const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS +const VIDEO_ABUSES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_ABUSES +const VIDEO_EVENTS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_EVENTS + +function isVideoAuthorValid (value) { + return isUserUsernameValid(value) +} + +function isVideoDateValid (value) { + return validator.isDate(value) +} + +function isVideoCategoryValid (value) { + return VIDEO_CATEGORIES[value] !== undefined +} + +function isVideoLicenceValid (value) { + return VIDEO_LICENCES[value] !== undefined +} + +function isVideoLanguageValid (value) { + return value === null || VIDEO_LANGUAGES[value] !== undefined +} + +function isVideoNSFWValid (value) { + return validator.isBoolean(value) +} + +function isVideoDescriptionValid (value) { + return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.DESCRIPTION) +} + +function isVideoDurationValid (value) { + return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION) +} + +function isVideoExtnameValid (value) { + return VIDEOS_CONSTRAINTS_FIELDS.EXTNAME.indexOf(value) !== -1 +} + +function isVideoInfoHashValid (value) { + return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.INFO_HASH) +} + +function isVideoNameValid (value) { + return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.NAME) +} + +function isVideoTagsValid (tags) { + return isArray(tags) && + validator.isInt(tags.length, VIDEOS_CONSTRAINTS_FIELDS.TAGS) && + tags.every(function (tag) { + return validator.isLength(tag, VIDEOS_CONSTRAINTS_FIELDS.TAG) + }) +} + +function isVideoThumbnailValid (value) { + return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.THUMBNAIL) +} + +function isVideoThumbnailDataValid (value) { + return validator.isByteLength(value, VIDEOS_CONSTRAINTS_FIELDS.THUMBNAIL_DATA) +} + +function isVideoRemoteIdValid (value) { + return validator.isUUID(value, 4) +} + +function isVideoAbuseReasonValid (value) { + return validator.isLength(value, VIDEO_ABUSES_CONSTRAINTS_FIELDS.REASON) +} + +function isVideoAbuseReporterUsernameValid (value) { + return isUserUsernameValid(value) +} + +function isVideoViewsValid (value) { + return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.VIEWS) +} + +function isVideoLikesValid (value) { + return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.LIKES) +} + +function isVideoDislikesValid (value) { + return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DISLIKES) +} + +function isVideoEventCountValid (value) { + return validator.isInt(value + '', VIDEO_EVENTS_CONSTRAINTS_FIELDS.COUNT) +} + +function isVideoRatingTypeValid (value) { + return values(VIDEO_RATE_TYPES).indexOf(value) !== -1 +} + +function isVideoFile (value, files) { + // Should have files + if (!files) return false + + // Should have videofile file + const videofile = files.videofile + if (!videofile || videofile.length === 0) return false + + // The file should exist + const file = videofile[0] + if (!file || !file.originalname) return false + + return new RegExp('^video/(webm|mp4|ogg)$', 'i').test(file.mimetype) +} + +// --------------------------------------------------------------------------- + +export { + isVideoAuthorValid, + isVideoDateValid, + isVideoCategoryValid, + isVideoLicenceValid, + isVideoLanguageValid, + isVideoNSFWValid, + isVideoDescriptionValid, + isVideoDurationValid, + isVideoInfoHashValid, + isVideoNameValid, + isVideoTagsValid, + isVideoThumbnailValid, + isVideoThumbnailDataValid, + isVideoExtnameValid, + isVideoRemoteIdValid, + isVideoAbuseReasonValid, + isVideoAbuseReporterUsernameValid, + isVideoFile, + isVideoViewsValid, + isVideoLikesValid, + isVideoRatingTypeValid, + isVideoDislikesValid, + isVideoEventCountValid +} diff --git a/server/helpers/database-utils.js b/server/helpers/database-utils.js deleted file mode 100644 index c72d19429..000000000 --- a/server/helpers/database-utils.js +++ /dev/null @@ -1,72 +0,0 @@ -'use strict' - -const retry = require('async/retry') - -const db = require('../initializers/database') -const logger = require('./logger') - -const utils = { - commitTransaction, - retryTransactionWrapper, - rollbackTransaction, - startSerializableTransaction, - transactionRetryer -} - -function commitTransaction (t, callback) { - return t.commit().asCallback(callback) -} - -function rollbackTransaction (err, t, callback) { - // Try to rollback transaction - if (t) { - // Do not catch err, report the original one - t.rollback().asCallback(function () { - return callback(err) - }) - } else { - return callback(err) - } -} - -// { arguments, errorMessage } -function retryTransactionWrapper (functionToRetry, options, finalCallback) { - const args = options.arguments ? options.arguments : [] - - utils.transactionRetryer( - function (callback) { - return functionToRetry.apply(this, args.concat([ callback ])) - }, - function (err) { - if (err) { - logger.error(options.errorMessage, { error: err }) - } - - // Do not return the error, continue the process - return finalCallback(null) - } - ) -} - -function transactionRetryer (func, callback) { - retry({ - times: 5, - - errorFilter: function (err) { - const willRetry = (err.name === 'SequelizeDatabaseError') - logger.debug('Maybe retrying the transaction function.', { willRetry }) - return willRetry - } - }, func, callback) -} - -function startSerializableTransaction (callback) { - db.sequelize.transaction({ isolationLevel: 'SERIALIZABLE' }).asCallback(function (err, t) { - // We force to return only two parameters - return callback(err, t) - }) -} - -// --------------------------------------------------------------------------- - -module.exports = utils diff --git a/server/helpers/database-utils.ts b/server/helpers/database-utils.ts new file mode 100644 index 000000000..b842ab9ec --- /dev/null +++ b/server/helpers/database-utils.ts @@ -0,0 +1,69 @@ +// TODO: import from ES6 when retry typing file will include errorFilter function +import retry = require('async/retry') + +const db = require('../initializers/database') +import { logger } from './logger' + +function commitTransaction (t, callback) { + return t.commit().asCallback(callback) +} + +function rollbackTransaction (err, t, callback) { + // Try to rollback transaction + if (t) { + // Do not catch err, report the original one + t.rollback().asCallback(function () { + return callback(err) + }) + } else { + return callback(err) + } +} + +// { arguments, errorMessage } +function retryTransactionWrapper (functionToRetry, options, finalCallback) { + const args = options.arguments ? options.arguments : [] + + transactionRetryer( + function (callback) { + return functionToRetry.apply(this, args.concat([ callback ])) + }, + function (err) { + if (err) { + logger.error(options.errorMessage, { error: err }) + } + + // Do not return the error, continue the process + return finalCallback(null) + } + ) +} + +function transactionRetryer (func, callback) { + retry({ + times: 5, + + errorFilter: function (err) { + const willRetry = (err.name === 'SequelizeDatabaseError') + logger.debug('Maybe retrying the transaction function.', { willRetry }) + return willRetry + } + }, func, callback) +} + +function startSerializableTransaction (callback) { + db.sequelize.transaction({ isolationLevel: 'SERIALIZABLE' }).asCallback(function (err, t) { + // We force to return only two parameters + return callback(err, t) + }) +} + +// --------------------------------------------------------------------------- + +export { + commitTransaction, + retryTransactionWrapper, + rollbackTransaction, + startSerializableTransaction, + transactionRetryer +} diff --git a/server/helpers/index.ts b/server/helpers/index.ts new file mode 100644 index 000000000..e56bd21ad --- /dev/null +++ b/server/helpers/index.ts @@ -0,0 +1,6 @@ +export * from './logger' +export * from './custom-validators' +export * from './database-utils' +export * from './peertube-crypto' +export * from './requests' +export * from './utils' diff --git a/server/helpers/logger.js b/server/helpers/logger.js deleted file mode 100644 index 281acedb8..000000000 --- a/server/helpers/logger.js +++ /dev/null @@ -1,49 +0,0 @@ -// Thanks http://tostring.it/2014/06/23/advanced-logging-with-nodejs/ -'use strict' - -const mkdirp = require('mkdirp') -const path = require('path') -const winston = require('winston') -winston.emitErrs = true - -const constants = require('../initializers/constants') - -const label = constants.CONFIG.WEBSERVER.HOSTNAME + ':' + constants.CONFIG.WEBSERVER.PORT - -// Create the directory if it does not exist -mkdirp.sync(constants.CONFIG.STORAGE.LOG_DIR) - -const logger = new winston.Logger({ - transports: [ - new winston.transports.File({ - level: 'debug', - filename: path.join(constants.CONFIG.STORAGE.LOG_DIR, 'all-logs.log'), - handleExceptions: true, - json: true, - maxsize: 5242880, - maxFiles: 5, - colorize: false, - prettyPrint: true - }), - new winston.transports.Console({ - level: 'debug', - label: label, - handleExceptions: true, - humanReadableUnhandledException: true, - json: false, - colorize: true, - prettyPrint: true - }) - ], - exitOnError: true -}) - -logger.stream = { - write: function (message, encoding) { - logger.info(message) - } -} - -// --------------------------------------------------------------------------- - -module.exports = logger diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts new file mode 100644 index 000000000..3c35e41e0 --- /dev/null +++ b/server/helpers/logger.ts @@ -0,0 +1,48 @@ +// Thanks http://tostring.it/2014/06/23/advanced-logging-with-nodejs/ +import mkdirp = require('mkdirp') +import path = require('path') +import winston = require('winston') + +// Do not use barrel (dependencies issues) +import { CONFIG } from '../initializers/constants' + +const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT + +// Create the directory if it does not exist +mkdirp.sync(CONFIG.STORAGE.LOG_DIR) + +const logger = new winston.Logger({ + transports: [ + new winston.transports.File({ + level: 'debug', + filename: path.join(CONFIG.STORAGE.LOG_DIR, 'all-logs.log'), + handleExceptions: true, + json: true, + maxsize: 5242880, + maxFiles: 5, + colorize: false, + prettyPrint: true + }), + new winston.transports.Console({ + level: 'debug', + label: label, + handleExceptions: true, + humanReadableUnhandledException: true, + json: false, + colorize: true, + prettyPrint: true + }) + ], + exitOnError: true +}) + +// TODO: useful? +// logger.stream = { +// write: function (message) { +// logger.info(message) +// } +// } + +// --------------------------------------------------------------------------- + +export { logger } diff --git a/server/helpers/peertube-crypto.js b/server/helpers/peertube-crypto.js deleted file mode 100644 index 55ae6fab3..000000000 --- a/server/helpers/peertube-crypto.js +++ /dev/null @@ -1,168 +0,0 @@ -'use strict' - -const crypto = require('crypto') -const bcrypt = require('bcrypt') -const fs = require('fs') -const openssl = require('openssl-wrapper') -const pathUtils = require('path') - -const constants = require('../initializers/constants') -const logger = require('./logger') - -const peertubeCrypto = { - checkSignature, - comparePassword, - createCertsIfNotExist, - cryptPassword, - getMyPrivateCert, - getMyPublicCert, - sign -} - -function checkSignature (publicKey, data, hexSignature) { - const verify = crypto.createVerify(constants.SIGNATURE_ALGORITHM) - - let dataString - if (typeof data === 'string') { - dataString = data - } else { - try { - dataString = JSON.stringify(data) - } catch (err) { - logger.error('Cannot check signature.', { error: err }) - return false - } - } - - verify.update(dataString, 'utf8') - - const isValid = verify.verify(publicKey, hexSignature, constants.SIGNATURE_ENCODING) - return isValid -} - -function sign (data) { - const sign = crypto.createSign(constants.SIGNATURE_ALGORITHM) - - let dataString - if (typeof data === 'string') { - dataString = data - } else { - try { - dataString = JSON.stringify(data) - } catch (err) { - logger.error('Cannot sign data.', { error: err }) - return '' - } - } - - sign.update(dataString, 'utf8') - - // TODO: make async - const certPath = pathUtils.join(constants.CONFIG.STORAGE.CERT_DIR, constants.PRIVATE_CERT_NAME) - const myKey = fs.readFileSync(certPath) - const signature = sign.sign(myKey, constants.SIGNATURE_ENCODING) - - return signature -} - -function comparePassword (plainPassword, hashPassword, callback) { - bcrypt.compare(plainPassword, hashPassword, function (err, isPasswordMatch) { - if (err) return callback(err) - - return callback(null, isPasswordMatch) - }) -} - -function createCertsIfNotExist (callback) { - certsExist(function (err, exist) { - if (err) return callback(err) - - if (exist === true) { - return callback(null) - } - - createCerts(function (err) { - return callback(err) - }) - }) -} - -function cryptPassword (password, callback) { - bcrypt.genSalt(constants.BCRYPT_SALT_SIZE, function (err, salt) { - if (err) return callback(err) - - bcrypt.hash(password, salt, function (err, hash) { - return callback(err, hash) - }) - }) -} - -function getMyPrivateCert (callback) { - const certPath = pathUtils.join(constants.CONFIG.STORAGE.CERT_DIR, constants.PRIVATE_CERT_NAME) - fs.readFile(certPath, 'utf8', callback) -} - -function getMyPublicCert (callback) { - const certPath = pathUtils.join(constants.CONFIG.STORAGE.CERT_DIR, constants.PUBLIC_CERT_NAME) - fs.readFile(certPath, 'utf8', callback) -} - -// --------------------------------------------------------------------------- - -module.exports = peertubeCrypto - -// --------------------------------------------------------------------------- - -function certsExist (callback) { - const certPath = pathUtils.join(constants.CONFIG.STORAGE.CERT_DIR, constants.PRIVATE_CERT_NAME) - fs.access(certPath, function (err) { - // If there is an error the certificates do not exist - const exists = !err - return callback(null, exists) - }) -} - -function createCerts (callback) { - certsExist(function (err, exist) { - if (err) return callback(err) - - if (exist === true) { - const string = 'Certs already exist.' - logger.warning(string) - return callback(new Error(string)) - } - - logger.info('Generating a RSA key...') - - const privateCertPath = pathUtils.join(constants.CONFIG.STORAGE.CERT_DIR, constants.PRIVATE_CERT_NAME) - const genRsaOptions = { - 'out': privateCertPath, - '2048': false - } - openssl.exec('genrsa', genRsaOptions, function (err) { - if (err) { - logger.error('Cannot create private key on this pod.') - return callback(err) - } - - logger.info('RSA key generated.') - logger.info('Managing public key...') - - const publicCertPath = pathUtils.join(constants.CONFIG.STORAGE.CERT_DIR, 'peertube.pub') - const rsaOptions = { - 'in': privateCertPath, - 'pubout': true, - 'out': publicCertPath - } - openssl.exec('rsa', rsaOptions, function (err) { - if (err) { - logger.error('Cannot create public key on this pod.') - return callback(err) - } - - logger.info('Public key managed.') - return callback(null) - }) - }) - }) -} diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts new file mode 100644 index 000000000..a4e9672e6 --- /dev/null +++ b/server/helpers/peertube-crypto.ts @@ -0,0 +1,171 @@ +import crypto = require('crypto') +import bcrypt = require('bcrypt') +import fs = require('fs') +import openssl = require('openssl-wrapper') +import { join } from 'path' + +import { + SIGNATURE_ALGORITHM, + SIGNATURE_ENCODING, + PRIVATE_CERT_NAME, + CONFIG, + BCRYPT_SALT_SIZE, + PUBLIC_CERT_NAME +} from '../initializers' +import { logger } from './logger' + +function checkSignature (publicKey, data, hexSignature) { + const verify = crypto.createVerify(SIGNATURE_ALGORITHM) + + let dataString + if (typeof data === 'string') { + dataString = data + } else { + try { + dataString = JSON.stringify(data) + } catch (err) { + logger.error('Cannot check signature.', { error: err }) + return false + } + } + + verify.update(dataString, 'utf8') + + const isValid = verify.verify(publicKey, hexSignature, SIGNATURE_ENCODING) + return isValid +} + +function sign (data) { + const sign = crypto.createSign(SIGNATURE_ALGORITHM) + + let dataString + if (typeof data === 'string') { + dataString = data + } else { + try { + dataString = JSON.stringify(data) + } catch (err) { + logger.error('Cannot sign data.', { error: err }) + return '' + } + } + + sign.update(dataString, 'utf8') + + // TODO: make async + const certPath = join(CONFIG.STORAGE.CERT_DIR, PRIVATE_CERT_NAME) + const myKey = fs.readFileSync(certPath) + const signature = sign.sign(myKey.toString(), SIGNATURE_ENCODING) + + return signature +} + +function comparePassword (plainPassword, hashPassword, callback) { + bcrypt.compare(plainPassword, hashPassword, function (err, isPasswordMatch) { + if (err) return callback(err) + + return callback(null, isPasswordMatch) + }) +} + +function createCertsIfNotExist (callback) { + certsExist(function (err, exist) { + if (err) return callback(err) + + if (exist === true) { + return callback(null) + } + + createCerts(function (err) { + return callback(err) + }) + }) +} + +function cryptPassword (password, callback) { + bcrypt.genSalt(BCRYPT_SALT_SIZE, function (err, salt) { + if (err) return callback(err) + + bcrypt.hash(password, salt, function (err, hash) { + return callback(err, hash) + }) + }) +} + +function getMyPrivateCert (callback) { + const certPath = join(CONFIG.STORAGE.CERT_DIR, PRIVATE_CERT_NAME) + fs.readFile(certPath, 'utf8', callback) +} + +function getMyPublicCert (callback) { + const certPath = join(CONFIG.STORAGE.CERT_DIR, PUBLIC_CERT_NAME) + fs.readFile(certPath, 'utf8', callback) +} + +// --------------------------------------------------------------------------- + +export { + checkSignature, + comparePassword, + createCertsIfNotExist, + cryptPassword, + getMyPrivateCert, + getMyPublicCert, + sign +} + +// --------------------------------------------------------------------------- + +function certsExist (callback) { + const certPath = join(CONFIG.STORAGE.CERT_DIR, PRIVATE_CERT_NAME) + fs.access(certPath, function (err) { + // If there is an error the certificates do not exist + const exists = !err + return callback(null, exists) + }) +} + +function createCerts (callback) { + certsExist(function (err, exist) { + if (err) return callback(err) + + if (exist === true) { + const string = 'Certs already exist.' + logger.warning(string) + return callback(new Error(string)) + } + + logger.info('Generating a RSA key...') + + const privateCertPath = join(CONFIG.STORAGE.CERT_DIR, PRIVATE_CERT_NAME) + const genRsaOptions = { + 'out': privateCertPath, + '2048': false + } + openssl.exec('genrsa', genRsaOptions, function (err) { + if (err) { + logger.error('Cannot create private key on this pod.') + return callback(err) + } + + logger.info('RSA key generated.') + logger.info('Managing public key...') + + const publicCertPath = join(CONFIG.STORAGE.CERT_DIR, 'peertube.pub') + const rsaOptions = { + 'in': privateCertPath, + 'pubout': true, + 'out': publicCertPath + } + openssl.exec('rsa', rsaOptions, function (err) { + if (err) { + logger.error('Cannot create public key on this pod.') + return callback(err) + } + + logger.info('Public key managed.') + return callback(null) + }) + }) + }) +} diff --git a/server/helpers/requests.js b/server/helpers/requests.js deleted file mode 100644 index efe056937..000000000 --- a/server/helpers/requests.js +++ /dev/null @@ -1,68 +0,0 @@ -'use strict' - -const replay = require('request-replay') -const request = require('request') - -const constants = require('../initializers/constants') -const peertubeCrypto = require('./peertube-crypto') - -const requests = { - makeRetryRequest, - makeSecureRequest -} - -function makeRetryRequest (params, callback) { - replay( - request(params, callback), - { - retries: constants.RETRY_REQUESTS, - factor: 3, - maxTimeout: Infinity, - errorCodes: [ 'EADDRINFO', 'ETIMEDOUT', 'ECONNRESET', 'ESOCKETTIMEDOUT', 'ENOTFOUND', 'ECONNREFUSED' ] - } - ) -} - -function makeSecureRequest (params, callback) { - const requestParams = { - url: constants.REMOTE_SCHEME.HTTP + '://' + params.toPod.host + params.path - } - - if (params.method !== 'POST') { - return callback(new Error('Cannot make a secure request with a non POST method.')) - } - - requestParams.json = {} - - // Add signature if it is specified in the params - if (params.sign === true) { - const host = constants.CONFIG.WEBSERVER.HOST - - let dataToSign - if (params.data) { - dataToSign = params.data - } else { - // We do not have data to sign so we just take our host - // It is not ideal but the connection should be in HTTPS - dataToSign = host - } - - requestParams.json.signature = { - host, // Which host we pretend to be - signature: peertubeCrypto.sign(dataToSign) - } - } - - // If there are data informations - if (params.data) { - requestParams.json.data = params.data - } - - console.log(requestParams.json.data) - - request.post(requestParams, callback) -} - -// --------------------------------------------------------------------------- - -module.exports = requests diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts new file mode 100644 index 000000000..8ded52972 --- /dev/null +++ b/server/helpers/requests.ts @@ -0,0 +1,65 @@ +import replay = require('request-replay') +import request = require('request') + +import { + RETRY_REQUESTS, + REMOTE_SCHEME, + CONFIG +} from '../initializers' +import { sign } from './peertube-crypto' + +function makeRetryRequest (params, callback) { + replay( + request(params, callback), + { + retries: RETRY_REQUESTS, + factor: 3, + maxTimeout: Infinity, + errorCodes: [ 'EADDRINFO', 'ETIMEDOUT', 'ECONNRESET', 'ESOCKETTIMEDOUT', 'ENOTFOUND', 'ECONNREFUSED' ] + } + ) +} + +function makeSecureRequest (params, callback) { + const requestParams = { + url: REMOTE_SCHEME.HTTP + '://' + params.toPod.host + params.path, + json: {} + } + + if (params.method !== 'POST') { + return callback(new Error('Cannot make a secure request with a non POST method.')) + } + + // Add signature if it is specified in the params + if (params.sign === true) { + const host = CONFIG.WEBSERVER.HOST + + let dataToSign + if (params.data) { + dataToSign = params.data + } else { + // We do not have data to sign so we just take our host + // It is not ideal but the connection should be in HTTPS + dataToSign = host + } + + requestParams.json['signature'] = { + host, // Which host we pretend to be + signature: sign(dataToSign) + } + } + + // If there are data informations + if (params.data) { + requestParams.json['data'] = params.data + } + + request.post(requestParams, callback) +} + +// --------------------------------------------------------------------------- + +export { + makeRetryRequest, + makeSecureRequest +} diff --git a/server/helpers/utils.js b/server/helpers/utils.js deleted file mode 100644 index 6d40e8f3f..000000000 --- a/server/helpers/utils.js +++ /dev/null @@ -1,58 +0,0 @@ -'use strict' - -const crypto = require('crypto') - -const logger = require('./logger') - -const utils = { - badRequest, - createEmptyCallback, - cleanForExit, - generateRandomString, - isTestInstance, - getFormatedObjects -} - -function badRequest (req, res, next) { - res.type('json').status(400).end() -} - -function generateRandomString (size, callback) { - crypto.pseudoRandomBytes(size, function (err, raw) { - if (err) return callback(err) - - callback(null, raw.toString('hex')) - }) -} - -function cleanForExit (webtorrentProcess) { - logger.info('Gracefully exiting.') - process.kill(-webtorrentProcess.pid) -} - -function createEmptyCallback () { - return function (err) { - if (err) logger.error('Error in empty callback.', { error: err }) - } -} - -function isTestInstance () { - return (process.env.NODE_ENV === 'test') -} - -function getFormatedObjects (objects, objectsTotal) { - const formatedObjects = [] - - objects.forEach(function (object) { - formatedObjects.push(object.toFormatedJSON()) - }) - - return { - total: objectsTotal, - data: formatedObjects - } -} - -// --------------------------------------------------------------------------- - -module.exports = utils diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts new file mode 100644 index 000000000..09c35a533 --- /dev/null +++ b/server/helpers/utils.ts @@ -0,0 +1,54 @@ +import { pseudoRandomBytes } from 'crypto' + +import { logger } from './logger' + +function badRequest (req, res, next) { + res.type('json').status(400).end() +} + +function generateRandomString (size, callback) { + pseudoRandomBytes(size, function (err, raw) { + if (err) return callback(err) + + callback(null, raw.toString('hex')) + }) +} + +function cleanForExit (webtorrentProcess) { + logger.info('Gracefully exiting.') + process.kill(-webtorrentProcess.pid) +} + +function createEmptyCallback () { + return function (err) { + if (err) logger.error('Error in empty callback.', { error: err }) + } +} + +function isTestInstance () { + return (process.env.NODE_ENV === 'test') +} + +function getFormatedObjects (objects, objectsTotal) { + const formatedObjects = [] + + objects.forEach(function (object) { + formatedObjects.push(object.toFormatedJSON()) + }) + + return { + total: objectsTotal, + data: formatedObjects + } +} + +// --------------------------------------------------------------------------- + +export { + badRequest, + createEmptyCallback, + cleanForExit, + generateRandomString, + isTestInstance, + getFormatedObjects +} diff --git a/server/initializers/checker.js b/server/initializers/checker.js deleted file mode 100644 index aa8dea4bf..000000000 --- a/server/initializers/checker.js +++ /dev/null @@ -1,88 +0,0 @@ -'use strict' - -const config = require('config') - -const constants = require('./constants') -const db = require('./database') - -const checker = { - checkConfig, - checkFFmpeg, - checkMissedConfig, - clientsExist, - usersExist -} - -// Some checks on configuration files -function checkConfig () { - if (config.has('webserver.host')) { - let errorMessage = '`host` config key was renamed to `hostname` but it seems you still have a `host` key in your configuration files!' - errorMessage += ' Please ensure to rename your `host` configuration to `hostname`.' - - return errorMessage - } - - return null -} - -// Check the config files -function checkMissedConfig () { - const required = [ 'listen.port', - 'webserver.https', 'webserver.hostname', 'webserver.port', - 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', - 'storage.certs', 'storage.videos', 'storage.logs', 'storage.thumbnails', 'storage.previews', - 'admin.email', 'signup.enabled', 'transcoding.enabled', 'transcoding.threads' - ] - const miss = [] - - for (const key of required) { - if (!config.has(key)) { - miss.push(key) - } - } - - return miss -} - -// Check the available codecs -function checkFFmpeg (callback) { - const Ffmpeg = require('fluent-ffmpeg') - - Ffmpeg.getAvailableCodecs(function (err, codecs) { - if (err) return callback(err) - if (constants.CONFIG.TRANSCODING.ENABLED === false) return callback(null) - - const canEncode = [ 'libx264' ] - canEncode.forEach(function (codec) { - if (codecs[codec] === undefined) { - return callback(new Error('Unknown codec ' + codec + ' in FFmpeg.')) - } - - if (codecs[codec].canEncode !== true) { - return callback(new Error('Unavailable encode codec ' + codec + ' in FFmpeg')) - } - }) - - return callback(null) - }) -} - -function clientsExist (callback) { - db.OAuthClient.countTotal(function (err, totalClients) { - if (err) return callback(err) - - return callback(null, totalClients !== 0) - }) -} - -function usersExist (callback) { - db.User.countTotal(function (err, totalUsers) { - if (err) return callback(err) - - return callback(null, totalUsers !== 0) - }) -} - -// --------------------------------------------------------------------------- - -module.exports = checker diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts new file mode 100644 index 000000000..370dff2d4 --- /dev/null +++ b/server/initializers/checker.ts @@ -0,0 +1,84 @@ +import config = require('config') + +const db = require('./database') +import { CONFIG } from './constants' + +// Some checks on configuration files +function checkConfig () { + if (config.has('webserver.host')) { + let errorMessage = '`host` config key was renamed to `hostname` but it seems you still have a `host` key in your configuration files!' + errorMessage += ' Please ensure to rename your `host` configuration to `hostname`.' + + return errorMessage + } + + return null +} + +// Check the config files +function checkMissedConfig () { + const required = [ 'listen.port', + 'webserver.https', 'webserver.hostname', 'webserver.port', + 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', + 'storage.certs', 'storage.videos', 'storage.logs', 'storage.thumbnails', 'storage.previews', + 'admin.email', 'signup.enabled', 'transcoding.enabled', 'transcoding.threads' + ] + const miss = [] + + for (const key of required) { + if (!config.has(key)) { + miss.push(key) + } + } + + return miss +} + +// Check the available codecs +function checkFFmpeg (callback) { + const Ffmpeg = require('fluent-ffmpeg') + + Ffmpeg.getAvailableCodecs(function (err, codecs) { + if (err) return callback(err) + if (CONFIG.TRANSCODING.ENABLED === false) return callback(null) + + const canEncode = [ 'libx264' ] + canEncode.forEach(function (codec) { + if (codecs[codec] === undefined) { + return callback(new Error('Unknown codec ' + codec + ' in FFmpeg.')) + } + + if (codecs[codec].canEncode !== true) { + return callback(new Error('Unavailable encode codec ' + codec + ' in FFmpeg')) + } + }) + + return callback(null) + }) +} + +function clientsExist (callback) { + db.OAuthClient.countTotal(function (err, totalClients) { + if (err) return callback(err) + + return callback(null, totalClients !== 0) + }) +} + +function usersExist (callback) { + db.User.countTotal(function (err, totalUsers) { + if (err) return callback(err) + + return callback(null, totalUsers !== 0) + }) +} + +// --------------------------------------------------------------------------- + +export { + checkConfig, + checkFFmpeg, + checkMissedConfig, + clientsExist, + usersExist +} diff --git a/server/initializers/constants.js b/server/initializers/constants.js deleted file mode 100644 index 87e9c8002..000000000 --- a/server/initializers/constants.js +++ /dev/null @@ -1,343 +0,0 @@ -'use strict' - -const config = require('config') -const path = require('path') - -// --------------------------------------------------------------------------- - -const LAST_MIGRATION_VERSION = 50 - -// --------------------------------------------------------------------------- - -// API version -const API_VERSION = 'v1' - -// Number of results by default for the pagination -const PAGINATION_COUNT_DEFAULT = 15 - -// Sortable columns per schema -const SEARCHABLE_COLUMNS = { - VIDEOS: [ 'name', 'magnetUri', 'host', 'author', 'tags' ] -} - -// Sortable columns per schema -const SORTABLE_COLUMNS = { - USERS: [ 'id', 'username', 'createdAt' ], - VIDEO_ABUSES: [ 'id', 'createdAt' ], - VIDEOS: [ 'name', 'duration', 'createdAt', 'views', 'likes' ] -} - -const OAUTH_LIFETIME = { - ACCESS_TOKEN: 3600 * 4, // 4 hours - REFRESH_TOKEN: 1209600 // 2 weeks -} - -// --------------------------------------------------------------------------- - -const CONFIG = { - LISTEN: { - PORT: config.get('listen.port') - }, - DATABASE: { - DBNAME: 'peertube' + config.get('database.suffix'), - HOSTNAME: config.get('database.hostname'), - PORT: config.get('database.port'), - USERNAME: config.get('database.username'), - PASSWORD: config.get('database.password') - }, - STORAGE: { - CERT_DIR: path.join(__dirname, '..', '..', config.get('storage.certs')), - LOG_DIR: path.join(__dirname, '..', '..', config.get('storage.logs')), - VIDEOS_DIR: path.join(__dirname, '..', '..', config.get('storage.videos')), - THUMBNAILS_DIR: path.join(__dirname, '..', '..', config.get('storage.thumbnails')), - PREVIEWS_DIR: path.join(__dirname, '..', '..', config.get('storage.previews')), - TORRENTS_DIR: path.join(__dirname, '..', '..', config.get('storage.torrents')) - }, - WEBSERVER: { - SCHEME: config.get('webserver.https') === true ? 'https' : 'http', - WS: config.get('webserver.https') === true ? 'wss' : 'ws', - HOSTNAME: config.get('webserver.hostname'), - PORT: config.get('webserver.port') - }, - ADMIN: { - EMAIL: config.get('admin.email') - }, - SIGNUP: { - ENABLED: config.get('signup.enabled') - }, - TRANSCODING: { - ENABLED: config.get('transcoding.enabled'), - THREADS: config.get('transcoding.threads') - } -} -CONFIG.WEBSERVER.URL = CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT -CONFIG.WEBSERVER.HOST = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT - -// --------------------------------------------------------------------------- - -const CONSTRAINTS_FIELDS = { - USERS: { - USERNAME: { min: 3, max: 20 }, // Length - PASSWORD: { min: 6, max: 255 } // Length - }, - VIDEO_ABUSES: { - REASON: { min: 2, max: 300 } // Length - }, - VIDEOS: { - NAME: { min: 3, max: 50 }, // Length - DESCRIPTION: { min: 3, max: 250 }, // Length - EXTNAME: [ '.mp4', '.ogv', '.webm' ], - INFO_HASH: { min: 40, max: 40 }, // Length, infohash is 20 bytes length but we represent it in hexa so 20 * 2 - DURATION: { min: 1, max: 7200 }, // Number - TAGS: { min: 0, max: 3 }, // Number of total tags - TAG: { min: 2, max: 10 }, // Length - THUMBNAIL: { min: 2, max: 30 }, - THUMBNAIL_DATA: { min: 0, max: 20000 }, // Bytes - VIEWS: { min: 0 }, - LIKES: { min: 0 }, - DISLIKES: { min: 0 } - }, - VIDEO_EVENTS: { - COUNT: { min: 0 } - } -} - -const VIDEO_RATE_TYPES = { - LIKE: 'like', - DISLIKE: 'dislike' -} - -const VIDEO_CATEGORIES = { - 1: 'Music', - 2: 'Films', - 3: 'Vehicles', - 4: 'Art', - 5: 'Sports', - 6: 'Travels', - 7: 'Gaming', - 8: 'People', - 9: 'Comedy', - 10: 'Entertainment', - 11: 'News', - 12: 'Howto', - 13: 'Education', - 14: 'Activism', - 15: 'Science & Technology', - 16: 'Animals', - 17: 'Kids', - 18: 'Food' -} - -// See https://creativecommons.org/licenses/?lang=en -const VIDEO_LICENCES = { - 1: 'Attribution', - 2: 'Attribution - Share Alike', - 3: 'Attribution - No Derivatives', - 4: 'Attribution - Non Commercial', - 5: 'Attribution - Non Commercial - Share Alike', - 6: 'Attribution - Non Commercial - No Derivatives', - 7: 'Public Domain Dedication' -} - -// See https://en.wikipedia.org/wiki/List_of_languages_by_number_of_native_speakers#Nationalencyklopedin -const VIDEO_LANGUAGES = { - 1: 'English', - 2: 'Spanish', - 3: 'Mandarin', - 4: 'Hindi', - 5: 'Arabic', - 6: 'Portuguese', - 7: 'Bengali', - 8: 'Russian', - 9: 'Japanese', - 10: 'Punjabi', - 11: 'German', - 12: 'Korean', - 13: 'French', - 14: 'Italien' -} - -// --------------------------------------------------------------------------- - -// Score a pod has when we create it as a friend -const FRIEND_SCORE = { - BASE: 100, - MAX: 1000 -} - -// --------------------------------------------------------------------------- - -// Number of points we add/remove from a friend after a successful/bad request -const PODS_SCORE = { - MALUS: -10, - BONUS: 10 -} - -// Time to wait between requests to the friends (10 min) -let REQUESTS_INTERVAL = 600000 - -// Number of requests in parallel we can make -const REQUESTS_IN_PARALLEL = 10 - -// To how many pods we send requests -const REQUESTS_LIMIT_PODS = 10 -// How many requests we send to a pod per interval -const REQUESTS_LIMIT_PER_POD = 5 - -const REQUESTS_VIDEO_QADU_LIMIT_PODS = 10 -// The QADU requests are not big -const REQUESTS_VIDEO_QADU_LIMIT_PER_POD = 50 - -const REQUESTS_VIDEO_EVENT_LIMIT_PODS = 10 -// The EVENTS requests are not big -const REQUESTS_VIDEO_EVENT_LIMIT_PER_POD = 50 - -// Number of requests to retry for replay requests module -const RETRY_REQUESTS = 5 - -const REQUEST_ENDPOINTS = { - VIDEOS: 'videos' -} - -const REQUEST_ENDPOINT_ACTIONS = {} -REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] = { - ADD: 'add', - UPDATE: 'update', - REMOVE: 'remove', - REPORT_ABUSE: 'report-abuse' -} - -const REQUEST_VIDEO_QADU_ENDPOINT = 'videos/qadu' -const REQUEST_VIDEO_EVENT_ENDPOINT = 'videos/events' - -const REQUEST_VIDEO_QADU_TYPES = { - LIKES: 'likes', - DISLIKES: 'dislikes', - VIEWS: 'views' -} - -const REQUEST_VIDEO_EVENT_TYPES = { - LIKES: 'likes', - DISLIKES: 'dislikes', - VIEWS: 'views' -} - -const REMOTE_SCHEME = { - HTTP: 'https', - WS: 'wss' -} - -const JOB_STATES = { - PENDING: 'pending', - PROCESSING: 'processing', - ERROR: 'error', - SUCCESS: 'success' -} -// How many maximum jobs we fetch from the database per cycle -const JOBS_FETCH_LIMIT_PER_CYCLE = 10 -const JOBS_CONCURRENCY = 1 -// 1 minutes -let JOBS_FETCHING_INTERVAL = 60000 - -// --------------------------------------------------------------------------- - -const PRIVATE_CERT_NAME = 'peertube.key.pem' -const PUBLIC_CERT_NAME = 'peertube.pub' -const SIGNATURE_ALGORITHM = 'RSA-SHA256' -const SIGNATURE_ENCODING = 'hex' - -// Password encryption -const BCRYPT_SALT_SIZE = 10 - -// --------------------------------------------------------------------------- - -// Express static paths (router) -const STATIC_PATHS = { - PREVIEWS: '/static/previews/', - THUMBNAILS: '/static/thumbnails/', - TORRENTS: '/static/torrents/', - WEBSEED: '/static/webseed/' -} - -// Cache control -let STATIC_MAX_AGE = '30d' - -// Videos thumbnail size -const THUMBNAILS_SIZE = '200x110' -const PREVIEWS_SIZE = '640x480' - -// --------------------------------------------------------------------------- - -const USER_ROLES = { - ADMIN: 'admin', - USER: 'user' -} - -// --------------------------------------------------------------------------- - -// Special constants for a test instance -if (isTestInstance() === true) { - CONSTRAINTS_FIELDS.VIDEOS.DURATION.max = 14 - FRIEND_SCORE.BASE = 20 - REQUESTS_INTERVAL = 10000 - JOBS_FETCHING_INTERVAL = 10000 - REMOTE_SCHEME.HTTP = 'http' - REMOTE_SCHEME.WS = 'ws' - STATIC_MAX_AGE = 0 -} - -// --------------------------------------------------------------------------- - -module.exports = { - API_VERSION, - BCRYPT_SALT_SIZE, - CONFIG, - CONSTRAINTS_FIELDS, - FRIEND_SCORE, - JOBS_FETCHING_INTERVAL, - JOB_STATES, - JOBS_CONCURRENCY, - JOBS_FETCH_LIMIT_PER_CYCLE, - LAST_MIGRATION_VERSION, - OAUTH_LIFETIME, - PAGINATION_COUNT_DEFAULT, - PODS_SCORE, - PREVIEWS_SIZE, - PRIVATE_CERT_NAME, - PUBLIC_CERT_NAME, - REMOTE_SCHEME, - REQUEST_ENDPOINT_ACTIONS, - REQUEST_ENDPOINTS, - REQUEST_VIDEO_EVENT_ENDPOINT, - REQUEST_VIDEO_EVENT_TYPES, - REQUEST_VIDEO_QADU_ENDPOINT, - REQUEST_VIDEO_QADU_TYPES, - REQUESTS_IN_PARALLEL, - REQUESTS_INTERVAL, - REQUESTS_LIMIT_PER_POD, - REQUESTS_LIMIT_PODS, - REQUESTS_VIDEO_EVENT_LIMIT_PER_POD, - REQUESTS_VIDEO_EVENT_LIMIT_PODS, - REQUESTS_VIDEO_QADU_LIMIT_PER_POD, - REQUESTS_VIDEO_QADU_LIMIT_PODS, - RETRY_REQUESTS, - SEARCHABLE_COLUMNS, - SIGNATURE_ALGORITHM, - SIGNATURE_ENCODING, - SORTABLE_COLUMNS, - STATIC_MAX_AGE, - STATIC_PATHS, - THUMBNAILS_SIZE, - USER_ROLES, - VIDEO_CATEGORIES, - VIDEO_LANGUAGES, - VIDEO_LICENCES, - VIDEO_RATE_TYPES -} - -// --------------------------------------------------------------------------- - -// This method exists in utils module but we want to let the constants module independent -function isTestInstance () { - return (process.env.NODE_ENV === 'test') -} diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts new file mode 100644 index 000000000..6bdc261ad --- /dev/null +++ b/server/initializers/constants.ts @@ -0,0 +1,343 @@ +import config = require('config') +import { join } from 'path' + +// --------------------------------------------------------------------------- + +const LAST_MIGRATION_VERSION = 50 + +// --------------------------------------------------------------------------- + +// API version +const API_VERSION = 'v1' + +// Number of results by default for the pagination +const PAGINATION_COUNT_DEFAULT = 15 + +// Sortable columns per schema +const SEARCHABLE_COLUMNS = { + VIDEOS: [ 'name', 'magnetUri', 'host', 'author', 'tags' ] +} + +// Sortable columns per schema +const SORTABLE_COLUMNS = { + USERS: [ 'id', 'username', 'createdAt' ], + VIDEO_ABUSES: [ 'id', 'createdAt' ], + VIDEOS: [ 'name', 'duration', 'createdAt', 'views', 'likes' ] +} + +const OAUTH_LIFETIME = { + ACCESS_TOKEN: 3600 * 4, // 4 hours + REFRESH_TOKEN: 1209600 // 2 weeks +} + +// --------------------------------------------------------------------------- + +const CONFIG = { + LISTEN: { + PORT: config.get('listen.port') + }, + DATABASE: { + DBNAME: 'peertube' + config.get('database.suffix'), + HOSTNAME: config.get('database.hostname'), + PORT: config.get('database.port'), + USERNAME: config.get('database.username'), + PASSWORD: config.get('database.password') + }, + STORAGE: { + CERT_DIR: join(__dirname, '..', '..', config.get('storage.certs')), + LOG_DIR: join(__dirname, '..', '..', config.get('storage.logs')), + VIDEOS_DIR: join(__dirname, '..', '..', config.get('storage.videos')), + THUMBNAILS_DIR: join(__dirname, '..', '..', config.get('storage.thumbnails')), + PREVIEWS_DIR: join(__dirname, '..', '..', config.get('storage.previews')), + TORRENTS_DIR: join(__dirname, '..', '..', config.get('storage.torrents')) + }, + WEBSERVER: { + SCHEME: config.get('webserver.https') === true ? 'https' : 'http', + WS: config.get('webserver.https') === true ? 'wss' : 'ws', + HOSTNAME: config.get('webserver.hostname'), + PORT: config.get('webserver.port'), + URL: '', + HOST: '' + }, + ADMIN: { + EMAIL: config.get('admin.email') + }, + SIGNUP: { + ENABLED: config.get('signup.enabled') + }, + TRANSCODING: { + ENABLED: config.get('transcoding.enabled'), + THREADS: config.get('transcoding.threads') + } +} +CONFIG.WEBSERVER.URL = CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT +CONFIG.WEBSERVER.HOST = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT + +// --------------------------------------------------------------------------- + +const CONSTRAINTS_FIELDS = { + USERS: { + USERNAME: { min: 3, max: 20 }, // Length + PASSWORD: { min: 6, max: 255 } // Length + }, + VIDEO_ABUSES: { + REASON: { min: 2, max: 300 } // Length + }, + VIDEOS: { + NAME: { min: 3, max: 50 }, // Length + DESCRIPTION: { min: 3, max: 250 }, // Length + EXTNAME: [ '.mp4', '.ogv', '.webm' ], + INFO_HASH: { min: 40, max: 40 }, // Length, infohash is 20 bytes length but we represent it in hexa so 20 * 2 + DURATION: { min: 1, max: 7200 }, // Number + TAGS: { min: 0, max: 3 }, // Number of total tags + TAG: { min: 2, max: 10 }, // Length + THUMBNAIL: { min: 2, max: 30 }, + THUMBNAIL_DATA: { min: 0, max: 20000 }, // Bytes + VIEWS: { min: 0 }, + LIKES: { min: 0 }, + DISLIKES: { min: 0 } + }, + VIDEO_EVENTS: { + COUNT: { min: 0 } + } +} + +const VIDEO_RATE_TYPES = { + LIKE: 'like', + DISLIKE: 'dislike' +} + +const VIDEO_CATEGORIES = { + 1: 'Music', + 2: 'Films', + 3: 'Vehicles', + 4: 'Art', + 5: 'Sports', + 6: 'Travels', + 7: 'Gaming', + 8: 'People', + 9: 'Comedy', + 10: 'Entertainment', + 11: 'News', + 12: 'Howto', + 13: 'Education', + 14: 'Activism', + 15: 'Science & Technology', + 16: 'Animals', + 17: 'Kids', + 18: 'Food' +} + +// See https://creativecommons.org/licenses/?lang=en +const VIDEO_LICENCES = { + 1: 'Attribution', + 2: 'Attribution - Share Alike', + 3: 'Attribution - No Derivatives', + 4: 'Attribution - Non Commercial', + 5: 'Attribution - Non Commercial - Share Alike', + 6: 'Attribution - Non Commercial - No Derivatives', + 7: 'Public Domain Dedication' +} + +// See https://en.wikipedia.org/wiki/List_of_languages_by_number_of_native_speakers#Nationalencyklopedin +const VIDEO_LANGUAGES = { + 1: 'English', + 2: 'Spanish', + 3: 'Mandarin', + 4: 'Hindi', + 5: 'Arabic', + 6: 'Portuguese', + 7: 'Bengali', + 8: 'Russian', + 9: 'Japanese', + 10: 'Punjabi', + 11: 'German', + 12: 'Korean', + 13: 'French', + 14: 'Italien' +} + +// --------------------------------------------------------------------------- + +// Score a pod has when we create it as a friend +const FRIEND_SCORE = { + BASE: 100, + MAX: 1000 +} + +// --------------------------------------------------------------------------- + +// Number of points we add/remove from a friend after a successful/bad request +const PODS_SCORE = { + MALUS: -10, + BONUS: 10 +} + +// Time to wait between requests to the friends (10 min) +let REQUESTS_INTERVAL = 600000 + +// Number of requests in parallel we can make +const REQUESTS_IN_PARALLEL = 10 + +// To how many pods we send requests +const REQUESTS_LIMIT_PODS = 10 +// How many requests we send to a pod per interval +const REQUESTS_LIMIT_PER_POD = 5 + +const REQUESTS_VIDEO_QADU_LIMIT_PODS = 10 +// The QADU requests are not big +const REQUESTS_VIDEO_QADU_LIMIT_PER_POD = 50 + +const REQUESTS_VIDEO_EVENT_LIMIT_PODS = 10 +// The EVENTS requests are not big +const REQUESTS_VIDEO_EVENT_LIMIT_PER_POD = 50 + +// Number of requests to retry for replay requests module +const RETRY_REQUESTS = 5 + +const REQUEST_ENDPOINTS = { + VIDEOS: 'videos' +} + +const REQUEST_ENDPOINT_ACTIONS = {} +REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] = { + ADD: 'add', + UPDATE: 'update', + REMOVE: 'remove', + REPORT_ABUSE: 'report-abuse' +} + +const REQUEST_VIDEO_QADU_ENDPOINT = 'videos/qadu' +const REQUEST_VIDEO_EVENT_ENDPOINT = 'videos/events' + +const REQUEST_VIDEO_QADU_TYPES = { + LIKES: 'likes', + DISLIKES: 'dislikes', + VIEWS: 'views' +} + +const REQUEST_VIDEO_EVENT_TYPES = { + LIKES: 'likes', + DISLIKES: 'dislikes', + VIEWS: 'views' +} + +const REMOTE_SCHEME = { + HTTP: 'https', + WS: 'wss' +} + +const JOB_STATES = { + PENDING: 'pending', + PROCESSING: 'processing', + ERROR: 'error', + SUCCESS: 'success' +} +// How many maximum jobs we fetch from the database per cycle +const JOBS_FETCH_LIMIT_PER_CYCLE = 10 +const JOBS_CONCURRENCY = 1 +// 1 minutes +let JOBS_FETCHING_INTERVAL = 60000 + +// --------------------------------------------------------------------------- + +const PRIVATE_CERT_NAME = 'peertube.key.pem' +const PUBLIC_CERT_NAME = 'peertube.pub' +const SIGNATURE_ALGORITHM = 'RSA-SHA256' +const SIGNATURE_ENCODING = 'hex' + +// Password encryption +const BCRYPT_SALT_SIZE = 10 + +// --------------------------------------------------------------------------- + +// Express static paths (router) +const STATIC_PATHS = { + PREVIEWS: '/static/previews/', + THUMBNAILS: '/static/thumbnails/', + TORRENTS: '/static/torrents/', + WEBSEED: '/static/webseed/' +} + +// Cache control +let STATIC_MAX_AGE = '30d' + +// Videos thumbnail size +const THUMBNAILS_SIZE = '200x110' +const PREVIEWS_SIZE = '640x480' + +// --------------------------------------------------------------------------- + +const USER_ROLES = { + ADMIN: 'admin', + USER: 'user' +} + +// --------------------------------------------------------------------------- + +// Special constants for a test instance +if (isTestInstance() === true) { + CONSTRAINTS_FIELDS.VIDEOS.DURATION.max = 14 + FRIEND_SCORE.BASE = 20 + REQUESTS_INTERVAL = 10000 + JOBS_FETCHING_INTERVAL = 10000 + REMOTE_SCHEME.HTTP = 'http' + REMOTE_SCHEME.WS = 'ws' + STATIC_MAX_AGE = '0' +} + +// --------------------------------------------------------------------------- + +export { + API_VERSION, + BCRYPT_SALT_SIZE, + CONFIG, + CONSTRAINTS_FIELDS, + FRIEND_SCORE, + JOBS_FETCHING_INTERVAL, + JOB_STATES, + JOBS_CONCURRENCY, + JOBS_FETCH_LIMIT_PER_CYCLE, + LAST_MIGRATION_VERSION, + OAUTH_LIFETIME, + PAGINATION_COUNT_DEFAULT, + PODS_SCORE, + PREVIEWS_SIZE, + PRIVATE_CERT_NAME, + PUBLIC_CERT_NAME, + REMOTE_SCHEME, + REQUEST_ENDPOINT_ACTIONS, + REQUEST_ENDPOINTS, + REQUEST_VIDEO_EVENT_ENDPOINT, + REQUEST_VIDEO_EVENT_TYPES, + REQUEST_VIDEO_QADU_ENDPOINT, + REQUEST_VIDEO_QADU_TYPES, + REQUESTS_IN_PARALLEL, + REQUESTS_INTERVAL, + REQUESTS_LIMIT_PER_POD, + REQUESTS_LIMIT_PODS, + REQUESTS_VIDEO_EVENT_LIMIT_PER_POD, + REQUESTS_VIDEO_EVENT_LIMIT_PODS, + REQUESTS_VIDEO_QADU_LIMIT_PER_POD, + REQUESTS_VIDEO_QADU_LIMIT_PODS, + RETRY_REQUESTS, + SEARCHABLE_COLUMNS, + SIGNATURE_ALGORITHM, + SIGNATURE_ENCODING, + SORTABLE_COLUMNS, + STATIC_MAX_AGE, + STATIC_PATHS, + THUMBNAILS_SIZE, + USER_ROLES, + VIDEO_CATEGORIES, + VIDEO_LANGUAGES, + VIDEO_LICENCES, + VIDEO_RATE_TYPES +} + +// --------------------------------------------------------------------------- + +// This method exists in utils module but we want to let the constants module independent +function isTestInstance () { + return (process.env.NODE_ENV === 'test') +} diff --git a/server/initializers/database.js b/server/initializers/database.js deleted file mode 100644 index 043152a0e..000000000 --- a/server/initializers/database.js +++ /dev/null @@ -1,77 +0,0 @@ -'use strict' - -const fs = require('fs') -const path = require('path') -const Sequelize = require('sequelize') - -const constants = require('../initializers/constants') -const logger = require('../helpers/logger') -const utils = require('../helpers/utils') - -const database = {} - -const dbname = constants.CONFIG.DATABASE.DBNAME -const username = constants.CONFIG.DATABASE.USERNAME -const password = constants.CONFIG.DATABASE.PASSWORD - -const sequelize = new Sequelize(dbname, username, password, { - dialect: 'postgres', - host: constants.CONFIG.DATABASE.HOSTNAME, - port: constants.CONFIG.DATABASE.PORT, - benchmark: utils.isTestInstance(), - - logging: function (message, benchmark) { - let newMessage = message - if (benchmark !== undefined) { - newMessage += ' | ' + benchmark + 'ms' - } - - logger.debug(newMessage) - } -}) - -database.sequelize = sequelize -database.Sequelize = Sequelize -database.init = init - -// --------------------------------------------------------------------------- - -module.exports = database - -// --------------------------------------------------------------------------- - -function init (silent, callback) { - if (!callback) { - callback = silent - silent = false - } - - if (!callback) callback = function () {} - - const modelDirectory = path.join(__dirname, '..', 'models') - fs.readdir(modelDirectory, function (err, files) { - if (err) throw err - - files.filter(function (file) { - // For all models but not utils.js - if (file === 'utils.js') return false - - return true - }) - .forEach(function (file) { - const model = sequelize.import(path.join(modelDirectory, file)) - - database[model.name] = model - }) - - Object.keys(database).forEach(function (modelName) { - if ('associate' in database[modelName]) { - database[modelName].associate(database) - } - }) - - if (!silent) logger.info('Database %s is ready.', dbname) - - return callback(null) - }) -} diff --git a/server/initializers/database.ts b/server/initializers/database.ts new file mode 100644 index 000000000..753a06669 --- /dev/null +++ b/server/initializers/database.ts @@ -0,0 +1,72 @@ +import fs = require('fs') +import { join } from 'path' +import Sequelize = require('sequelize') + +import { CONFIG } from './constants' +// Do not use barrel, we need to load database first +import { logger } from '../helpers/logger' +import { isTestInstance } from '../helpers/utils' + +const dbname = CONFIG.DATABASE.DBNAME +const username = CONFIG.DATABASE.USERNAME +const password = CONFIG.DATABASE.PASSWORD + +const database: any = {} + +const sequelize = new Sequelize(dbname, username, password, { + dialect: 'postgres', + host: CONFIG.DATABASE.HOSTNAME, + port: CONFIG.DATABASE.PORT, + benchmark: isTestInstance(), + + logging: function (message, benchmark) { + let newMessage = message + if (benchmark !== undefined) { + newMessage += ' | ' + benchmark + 'ms' + } + + logger.debug(newMessage) + } +}) + +database.sequelize = sequelize + +database.init = function (silent, callback) { + if (!callback) { + callback = silent + silent = false + } + + if (!callback) callback = function () { /* empty */ } + + const modelDirectory = join(__dirname, '..', 'models') + fs.readdir(modelDirectory, function (err, files) { + if (err) throw err + + files.filter(function (file) { + // For all models but not utils.js + if (file === 'utils.js') return false + + return true + }) + .forEach(function (file) { + const model = sequelize.import(join(modelDirectory, file)) + + database[model['name']] = model + }) + + Object.keys(database).forEach(function (modelName) { + if ('associate' in database[modelName]) { + database[modelName].associate(database) + } + }) + + if (!silent) logger.info('Database %s is ready.', dbname) + + return callback(null) + }) +} + +// --------------------------------------------------------------------------- + +module.exports = database diff --git a/server/initializers/index.ts b/server/initializers/index.ts new file mode 100644 index 000000000..b8400ff84 --- /dev/null +++ b/server/initializers/index.ts @@ -0,0 +1,6 @@ +// Constants first, databse in second! +export * from './constants' +export * from './database' +export * from './checker' +export * from './installer' +export * from './migrator' diff --git a/server/initializers/installer.js b/server/initializers/installer.js deleted file mode 100644 index 837a987dd..000000000 --- a/server/initializers/installer.js +++ /dev/null @@ -1,134 +0,0 @@ -'use strict' - -const config = require('config') -const each = require('async/each') -const mkdirp = require('mkdirp') -const passwordGenerator = require('password-generator') -const path = require('path') -const series = require('async/series') - -const checker = require('./checker') -const constants = require('./constants') -const db = require('./database') -const logger = require('../helpers/logger') -const peertubeCrypto = require('../helpers/peertube-crypto') - -const installer = { - installApplication -} - -function installApplication (callback) { - series([ - function createDatabase (callbackAsync) { - db.sequelize.sync().asCallback(callbackAsync) - // db.sequelize.sync({ force: true }).asCallback(callbackAsync) - }, - - function createDirectories (callbackAsync) { - createDirectoriesIfNotExist(callbackAsync) - }, - - function createCertificates (callbackAsync) { - peertubeCrypto.createCertsIfNotExist(callbackAsync) - }, - - function createOAuthClient (callbackAsync) { - createOAuthClientIfNotExist(callbackAsync) - }, - - function createOAuthUser (callbackAsync) { - createOAuthAdminIfNotExist(callbackAsync) - } - ], callback) -} - -// --------------------------------------------------------------------------- - -module.exports = installer - -// --------------------------------------------------------------------------- - -function createDirectoriesIfNotExist (callback) { - const storages = config.get('storage') - - each(Object.keys(storages), function (key, callbackEach) { - const dir = storages[key] - mkdirp(path.join(__dirname, '..', '..', dir), callbackEach) - }, callback) -} - -function createOAuthClientIfNotExist (callback) { - checker.clientsExist(function (err, exist) { - if (err) return callback(err) - - // Nothing to do, clients already exist - if (exist === true) return callback(null) - - logger.info('Creating a default OAuth Client.') - - const id = passwordGenerator(32, false, /[a-z0-9]/) - const secret = passwordGenerator(32, false, /[a-zA-Z0-9]/) - const client = db.OAuthClient.build({ - clientId: id, - clientSecret: secret, - grants: [ 'password', 'refresh_token' ] - }) - - client.save().asCallback(function (err, createdClient) { - if (err) return callback(err) - - logger.info('Client id: ' + createdClient.clientId) - logger.info('Client secret: ' + createdClient.clientSecret) - - return callback(null) - }) - }) -} - -function createOAuthAdminIfNotExist (callback) { - checker.usersExist(function (err, exist) { - if (err) return callback(err) - - // Nothing to do, users already exist - if (exist === true) return callback(null) - - logger.info('Creating the administrator.') - - const username = 'root' - const role = constants.USER_ROLES.ADMIN - const email = constants.CONFIG.ADMIN.EMAIL - const createOptions = {} - let password = '' - - // Do not generate a random password for tests - if (process.env.NODE_ENV === 'test') { - password = 'test' - - if (process.env.NODE_APP_INSTANCE) { - password += process.env.NODE_APP_INSTANCE - } - - // Our password is weak so do not validate it - createOptions.validate = false - } else { - password = passwordGenerator(8, true) - } - - const userData = { - username, - email, - password, - role - } - - db.User.create(userData, createOptions).asCallback(function (err, createdUser) { - if (err) return callback(err) - - logger.info('Username: ' + username) - logger.info('User password: ' + password) - - logger.info('Creating Application table.') - db.Application.create({ migrationVersion: constants.LAST_MIGRATION_VERSION }).asCallback(callback) - }) - }) -} diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts new file mode 100644 index 000000000..cd1404d48 --- /dev/null +++ b/server/initializers/installer.ts @@ -0,0 +1,128 @@ +import { join } from 'path' +import config = require('config') +import { each, series } from 'async' +import mkdirp = require('mkdirp') +import passwordGenerator = require('password-generator') + +const db = require('./database') +import { USER_ROLES, CONFIG, LAST_MIGRATION_VERSION } from './constants' +import { clientsExist, usersExist } from './checker' +import { logger, createCertsIfNotExist } from '../helpers' + +function installApplication (callback) { + series([ + function createDatabase (callbackAsync) { + db.sequelize.sync().asCallback(callbackAsync) + // db.sequelize.sync({ force: true }).asCallback(callbackAsync) + }, + + function createDirectories (callbackAsync) { + createDirectoriesIfNotExist(callbackAsync) + }, + + function createCertificates (callbackAsync) { + createCertsIfNotExist(callbackAsync) + }, + + function createOAuthClient (callbackAsync) { + createOAuthClientIfNotExist(callbackAsync) + }, + + function createOAuthUser (callbackAsync) { + createOAuthAdminIfNotExist(callbackAsync) + } + ], callback) +} + +// --------------------------------------------------------------------------- + +export { + installApplication +} + +// --------------------------------------------------------------------------- + +function createDirectoriesIfNotExist (callback) { + const storages = config.get('storage') + + each(Object.keys(storages), function (key, callbackEach) { + const dir = storages[key] + mkdirp(join(__dirname, '..', '..', dir), callbackEach) + }, callback) +} + +function createOAuthClientIfNotExist (callback) { + clientsExist(function (err, exist) { + if (err) return callback(err) + + // Nothing to do, clients already exist + if (exist === true) return callback(null) + + logger.info('Creating a default OAuth Client.') + + const id = passwordGenerator(32, false, /[a-z0-9]/) + const secret = passwordGenerator(32, false, /[a-zA-Z0-9]/) + const client = db.OAuthClient.build({ + clientId: id, + clientSecret: secret, + grants: [ 'password', 'refresh_token' ] + }) + + client.save().asCallback(function (err, createdClient) { + if (err) return callback(err) + + logger.info('Client id: ' + createdClient.clientId) + logger.info('Client secret: ' + createdClient.clientSecret) + + return callback(null) + }) + }) +} + +function createOAuthAdminIfNotExist (callback) { + usersExist(function (err, exist) { + if (err) return callback(err) + + // Nothing to do, users already exist + if (exist === true) return callback(null) + + logger.info('Creating the administrator.') + + const username = 'root' + const role = USER_ROLES.ADMIN + const email = CONFIG.ADMIN.EMAIL + const createOptions: { validate?: boolean } = {} + let password = '' + + // Do not generate a random password for tests + if (process.env.NODE_ENV === 'test') { + password = 'test' + + if (process.env.NODE_APP_INSTANCE) { + password += process.env.NODE_APP_INSTANCE + } + + // Our password is weak so do not validate it + createOptions.validate = false + } else { + password = passwordGenerator(8, true) + } + + const userData = { + username, + email, + password, + role + } + + db.User.create(userData, createOptions).asCallback(function (err, createdUser) { + if (err) return callback(err) + + logger.info('Username: ' + username) + logger.info('User password: ' + password) + + logger.info('Creating Application table.') + db.Application.create({ migrationVersion: LAST_MIGRATION_VERSION }).asCallback(callback) + }) + }) +} diff --git a/server/initializers/migrations/0005-email-pod.js b/server/initializers/migrations/0005-email-pod.js deleted file mode 100644 index 9bbb354bf..000000000 --- a/server/initializers/migrations/0005-email-pod.js +++ /dev/null @@ -1,41 +0,0 @@ -'use strict' - -const waterfall = require('async/waterfall') - -// utils = { transaction, queryInterface, sequelize, Sequelize } -exports.up = function (utils, finalCallback) { - const q = utils.queryInterface - const Sequelize = utils.Sequelize - - const data = { - type: Sequelize.STRING(400), - allowNull: false, - defaultValue: '' - } - - waterfall([ - - function addEmailColumn (callback) { - q.addColumn('Pods', 'email', data, { transaction: utils.transaction }).asCallback(function (err) { - return callback(err) - }) - }, - - function updateWithFakeEmails (callback) { - const query = 'UPDATE "Pods" SET "email" = \'dummy@example.com\'' - utils.sequelize.query(query, { transaction: utils.transaction }).asCallback(function (err) { - return callback(err) - }) - }, - - function nullOnDefault (callback) { - data.defaultValue = null - - q.changeColumn('Pods', 'email', data, { transaction: utils.transaction }).asCallback(callback) - } - ], finalCallback) -} - -exports.down = function (options, callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0005-email-pod.ts b/server/initializers/migrations/0005-email-pod.ts new file mode 100644 index 000000000..a9200c47f --- /dev/null +++ b/server/initializers/migrations/0005-email-pod.ts @@ -0,0 +1,44 @@ +import { waterfall } from 'async' + +// utils = { transaction, queryInterface, sequelize, Sequelize } +function up (utils, finalCallback) { + const q = utils.queryInterface + const Sequelize = utils.Sequelize + + const data = { + type: Sequelize.STRING(400), + allowNull: false, + defaultValue: '' + } + + waterfall([ + + function addEmailColumn (callback) { + q.addColumn('Pods', 'email', data, { transaction: utils.transaction }).asCallback(function (err) { + return callback(err) + }) + }, + + function updateWithFakeEmails (callback) { + const query = 'UPDATE "Pods" SET "email" = \'dummy@example.com\'' + utils.sequelize.query(query, { transaction: utils.transaction }).asCallback(function (err) { + return callback(err) + }) + }, + + function nullOnDefault (callback) { + data.defaultValue = null + + q.changeColumn('Pods', 'email', data, { transaction: utils.transaction }).asCallback(callback) + } + ], finalCallback) +} + +function down (options, callback) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/initializers/migrations/0010-email-user.js b/server/initializers/migrations/0010-email-user.js deleted file mode 100644 index 1ab27133a..000000000 --- a/server/initializers/migrations/0010-email-user.js +++ /dev/null @@ -1,41 +0,0 @@ -'use strict' - -const waterfall = require('async/waterfall') - -// utils = { transaction, queryInterface, sequelize, Sequelize } -exports.up = function (utils, finalCallback) { - const q = utils.queryInterface - const Sequelize = utils.Sequelize - - const data = { - type: Sequelize.STRING(400), - allowNull: false, - defaultValue: '' - } - - waterfall([ - - function addEmailColumn (callback) { - q.addColumn('Users', 'email', data, { transaction: utils.transaction }).asCallback(function (err) { - return callback(err) - }) - }, - - function updateWithFakeEmails (callback) { - const query = 'UPDATE "Users" SET "email" = CONCAT("username", \'@example.com\')' - utils.sequelize.query(query, { transaction: utils.transaction }).asCallback(function (err) { - return callback(err) - }) - }, - - function nullOnDefault (callback) { - data.defaultValue = null - - q.changeColumn('Users', 'email', data, { transaction: utils.transaction }).asCallback(callback) - } - ], finalCallback) -} - -exports.down = function (options, callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0010-email-user.ts b/server/initializers/migrations/0010-email-user.ts new file mode 100644 index 000000000..4b5d29394 --- /dev/null +++ b/server/initializers/migrations/0010-email-user.ts @@ -0,0 +1,44 @@ +import { waterfall } from 'async' + +// utils = { transaction, queryInterface, sequelize, Sequelize } +function up (utils, finalCallback) { + const q = utils.queryInterface + const Sequelize = utils.Sequelize + + const data = { + type: Sequelize.STRING(400), + allowNull: false, + defaultValue: '' + } + + waterfall([ + + function addEmailColumn (callback) { + q.addColumn('Users', 'email', data, { transaction: utils.transaction }).asCallback(function (err) { + return callback(err) + }) + }, + + function updateWithFakeEmails (callback) { + const query = 'UPDATE "Users" SET "email" = CONCAT("username", \'@example.com\')' + utils.sequelize.query(query, { transaction: utils.transaction }).asCallback(function (err) { + return callback(err) + }) + }, + + function nullOnDefault (callback) { + data.defaultValue = null + + q.changeColumn('Users', 'email', data, { transaction: utils.transaction }).asCallback(callback) + } + ], finalCallback) +} + +function down (options, callback) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/initializers/migrations/0015-video-views.js b/server/initializers/migrations/0015-video-views.js deleted file mode 100644 index ae49fe73c..000000000 --- a/server/initializers/migrations/0015-video-views.js +++ /dev/null @@ -1,19 +0,0 @@ -'use strict' - -// utils = { transaction, queryInterface, sequelize, Sequelize } -exports.up = function (utils, finalCallback) { - const q = utils.queryInterface - const Sequelize = utils.Sequelize - - const data = { - type: Sequelize.INTEGER, - allowNull: false, - defaultValue: 0 - } - - q.addColumn('Videos', 'views', data, { transaction: utils.transaction }).asCallback(finalCallback) -} - -exports.down = function (options, callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0015-video-views.ts b/server/initializers/migrations/0015-video-views.ts new file mode 100644 index 000000000..e70869404 --- /dev/null +++ b/server/initializers/migrations/0015-video-views.ts @@ -0,0 +1,22 @@ +// utils = { transaction, queryInterface, sequelize, Sequelize } +function up (utils, finalCallback) { + const q = utils.queryInterface + const Sequelize = utils.Sequelize + + const data = { + type: Sequelize.INTEGER, + allowNull: false, + defaultValue: 0 + } + + q.addColumn('Videos', 'views', data, { transaction: utils.transaction }).asCallback(finalCallback) +} + +function down (options, callback) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/initializers/migrations/0020-video-likes.js b/server/initializers/migrations/0020-video-likes.js deleted file mode 100644 index 6db62cb90..000000000 --- a/server/initializers/migrations/0020-video-likes.js +++ /dev/null @@ -1,19 +0,0 @@ -'use strict' - -// utils = { transaction, queryInterface, sequelize, Sequelize } -exports.up = function (utils, finalCallback) { - const q = utils.queryInterface - const Sequelize = utils.Sequelize - - const data = { - type: Sequelize.INTEGER, - allowNull: false, - defaultValue: 0 - } - - q.addColumn('Videos', 'likes', data, { transaction: utils.transaction }).asCallback(finalCallback) -} - -exports.down = function (options, callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0020-video-likes.ts b/server/initializers/migrations/0020-video-likes.ts new file mode 100644 index 000000000..e435d0657 --- /dev/null +++ b/server/initializers/migrations/0020-video-likes.ts @@ -0,0 +1,22 @@ +// utils = { transaction, queryInterface, sequelize, Sequelize } +function up (utils, finalCallback) { + const q = utils.queryInterface + const Sequelize = utils.Sequelize + + const data = { + type: Sequelize.INTEGER, + allowNull: false, + defaultValue: 0 + } + + q.addColumn('Videos', 'likes', data, { transaction: utils.transaction }).asCallback(finalCallback) +} + +function down (options, callback) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/initializers/migrations/0025-video-dislikes.js b/server/initializers/migrations/0025-video-dislikes.js deleted file mode 100644 index 40d2e7351..000000000 --- a/server/initializers/migrations/0025-video-dislikes.js +++ /dev/null @@ -1,19 +0,0 @@ -'use strict' - -// utils = { transaction, queryInterface, sequelize, Sequelize } -exports.up = function (utils, finalCallback) { - const q = utils.queryInterface - const Sequelize = utils.Sequelize - - const data = { - type: Sequelize.INTEGER, - allowNull: false, - defaultValue: 0 - } - - q.addColumn('Videos', 'dislikes', data, { transaction: utils.transaction }).asCallback(finalCallback) -} - -exports.down = function (options, callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0025-video-dislikes.ts b/server/initializers/migrations/0025-video-dislikes.ts new file mode 100644 index 000000000..57e54e904 --- /dev/null +++ b/server/initializers/migrations/0025-video-dislikes.ts @@ -0,0 +1,22 @@ +// utils = { transaction, queryInterface, sequelize, Sequelize } +function up (utils, finalCallback) { + const q = utils.queryInterface + const Sequelize = utils.Sequelize + + const data = { + type: Sequelize.INTEGER, + allowNull: false, + defaultValue: 0 + } + + q.addColumn('Videos', 'dislikes', data, { transaction: utils.transaction }).asCallback(finalCallback) +} + +function down (options, callback) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/initializers/migrations/0030-video-category.js b/server/initializers/migrations/0030-video-category.js deleted file mode 100644 index ada95b2fe..000000000 --- a/server/initializers/migrations/0030-video-category.js +++ /dev/null @@ -1,34 +0,0 @@ -'use strict' - -const waterfall = require('async/waterfall') - -// utils = { transaction, queryInterface, sequelize, Sequelize } -exports.up = function (utils, finalCallback) { - const q = utils.queryInterface - const Sequelize = utils.Sequelize - - const data = { - type: Sequelize.INTEGER, - allowNull: false, - defaultValue: 0 - } - - waterfall([ - - function addCategoryColumn (callback) { - q.addColumn('Videos', 'category', data, { transaction: utils.transaction }).asCallback(function (err) { - return callback(err) - }) - }, - - function nullOnDefault (callback) { - data.defaultValue = null - - q.changeColumn('Videos', 'category', data, { transaction: utils.transaction }).asCallback(callback) - } - ], finalCallback) -} - -exports.down = function (options, callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0030-video-category.ts b/server/initializers/migrations/0030-video-category.ts new file mode 100644 index 000000000..1073f449c --- /dev/null +++ b/server/initializers/migrations/0030-video-category.ts @@ -0,0 +1,37 @@ +import { waterfall } from 'async' + +// utils = { transaction, queryInterface, sequelize, Sequelize } +function up (utils, finalCallback) { + const q = utils.queryInterface + const Sequelize = utils.Sequelize + + const data = { + type: Sequelize.INTEGER, + allowNull: false, + defaultValue: 0 + } + + waterfall([ + + function addCategoryColumn (callback) { + q.addColumn('Videos', 'category', data, { transaction: utils.transaction }).asCallback(function (err) { + return callback(err) + }) + }, + + function nullOnDefault (callback) { + data.defaultValue = null + + q.changeColumn('Videos', 'category', data, { transaction: utils.transaction }).asCallback(callback) + } + ], finalCallback) +} + +function down (options, callback) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/initializers/migrations/0035-video-licence.js b/server/initializers/migrations/0035-video-licence.js deleted file mode 100644 index 9cf75858d..000000000 --- a/server/initializers/migrations/0035-video-licence.js +++ /dev/null @@ -1,34 +0,0 @@ -'use strict' - -const waterfall = require('async/waterfall') - -// utils = { transaction, queryInterface, sequelize, Sequelize } -exports.up = function (utils, finalCallback) { - const q = utils.queryInterface - const Sequelize = utils.Sequelize - - const data = { - type: Sequelize.INTEGER, - allowNull: false, - defaultValue: 0 - } - - waterfall([ - - function addLicenceColumn (callback) { - q.addColumn('Videos', 'licence', data, { transaction: utils.transaction }).asCallback(function (err) { - return callback(err) - }) - }, - - function nullOnDefault (callback) { - data.defaultValue = null - - q.changeColumn('Videos', 'licence', data, { transaction: utils.transaction }).asCallback(callback) - } - ], finalCallback) -} - -exports.down = function (options, callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0035-video-licence.ts b/server/initializers/migrations/0035-video-licence.ts new file mode 100644 index 000000000..9316b3c37 --- /dev/null +++ b/server/initializers/migrations/0035-video-licence.ts @@ -0,0 +1,37 @@ +import { waterfall } from 'async' + +// utils = { transaction, queryInterface, sequelize, Sequelize } +function up (utils, finalCallback) { + const q = utils.queryInterface + const Sequelize = utils.Sequelize + + const data = { + type: Sequelize.INTEGER, + allowNull: false, + defaultValue: 0 + } + + waterfall([ + + function addLicenceColumn (callback) { + q.addColumn('Videos', 'licence', data, { transaction: utils.transaction }).asCallback(function (err) { + return callback(err) + }) + }, + + function nullOnDefault (callback) { + data.defaultValue = null + + q.changeColumn('Videos', 'licence', data, { transaction: utils.transaction }).asCallback(callback) + } + ], finalCallback) +} + +function down (options, callback) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/initializers/migrations/0040-video-nsfw.js b/server/initializers/migrations/0040-video-nsfw.js deleted file mode 100644 index 7f3692b28..000000000 --- a/server/initializers/migrations/0040-video-nsfw.js +++ /dev/null @@ -1,34 +0,0 @@ -'use strict' - -const waterfall = require('async/waterfall') - -// utils = { transaction, queryInterface, sequelize, Sequelize } -exports.up = function (utils, finalCallback) { - const q = utils.queryInterface - const Sequelize = utils.Sequelize - - const data = { - type: Sequelize.BOOLEAN, - allowNull: false, - defaultValue: false - } - - waterfall([ - - function addNSFWColumn (callback) { - q.addColumn('Videos', 'nsfw', data, { transaction: utils.transaction }).asCallback(function (err) { - return callback(err) - }) - }, - - function nullOnDefault (callback) { - data.defaultValue = null - - q.changeColumn('Videos', 'nsfw', data, { transaction: utils.transaction }).asCallback(callback) - } - ], finalCallback) -} - -exports.down = function (options, callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0040-video-nsfw.ts b/server/initializers/migrations/0040-video-nsfw.ts new file mode 100644 index 000000000..c61f496f1 --- /dev/null +++ b/server/initializers/migrations/0040-video-nsfw.ts @@ -0,0 +1,37 @@ +import { waterfall } from 'async' + +// utils = { transaction, queryInterface, sequelize, Sequelize } +function up (utils, finalCallback) { + const q = utils.queryInterface + const Sequelize = utils.Sequelize + + const data = { + type: Sequelize.BOOLEAN, + allowNull: false, + defaultValue: false + } + + waterfall([ + + function addNSFWColumn (callback) { + q.addColumn('Videos', 'nsfw', data, { transaction: utils.transaction }).asCallback(function (err) { + return callback(err) + }) + }, + + function nullOnDefault (callback) { + data.defaultValue = null + + q.changeColumn('Videos', 'nsfw', data, { transaction: utils.transaction }).asCallback(callback) + } + ], finalCallback) +} + +function down (options, callback) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/initializers/migrations/0045-user-display-nsfw.js b/server/initializers/migrations/0045-user-display-nsfw.js deleted file mode 100644 index 03624e593..000000000 --- a/server/initializers/migrations/0045-user-display-nsfw.js +++ /dev/null @@ -1,19 +0,0 @@ -'use strict' - -// utils = { transaction, queryInterface, sequelize, Sequelize } -exports.up = function (utils, finalCallback) { - const q = utils.queryInterface - const Sequelize = utils.Sequelize - - const data = { - type: Sequelize.BOOLEAN, - allowNull: false, - defaultValue: false - } - - q.addColumn('Users', 'displayNSFW', data, { transaction: utils.transaction }).asCallback(finalCallback) -} - -exports.down = function (options, callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0045-user-display-nsfw.ts b/server/initializers/migrations/0045-user-display-nsfw.ts new file mode 100644 index 000000000..1ca317795 --- /dev/null +++ b/server/initializers/migrations/0045-user-display-nsfw.ts @@ -0,0 +1,22 @@ +// utils = { transaction, queryInterface, sequelize, Sequelize } +function up (utils, finalCallback) { + const q = utils.queryInterface + const Sequelize = utils.Sequelize + + const data = { + type: Sequelize.BOOLEAN, + allowNull: false, + defaultValue: false + } + + q.addColumn('Users', 'displayNSFW', data, { transaction: utils.transaction }).asCallback(finalCallback) +} + +function down (options, callback) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/initializers/migrations/0050-video-language.js b/server/initializers/migrations/0050-video-language.js deleted file mode 100644 index 1c978758d..000000000 --- a/server/initializers/migrations/0050-video-language.js +++ /dev/null @@ -1,19 +0,0 @@ -'use strict' - -// utils = { transaction, queryInterface, sequelize, Sequelize } -exports.up = function (utils, finalCallback) { - const q = utils.queryInterface - const Sequelize = utils.Sequelize - - const data = { - type: Sequelize.INTEGER, - allowNull: true, - defaultValue: null - } - - q.addColumn('Videos', 'language', data, { transaction: utils.transaction }).asCallback(finalCallback) -} - -exports.down = function (options, callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0050-video-language.ts b/server/initializers/migrations/0050-video-language.ts new file mode 100644 index 000000000..95d0a473a --- /dev/null +++ b/server/initializers/migrations/0050-video-language.ts @@ -0,0 +1,22 @@ +// utils = { transaction, queryInterface, sequelize, Sequelize } +function up (utils, finalCallback) { + const q = utils.queryInterface + const Sequelize = utils.Sequelize + + const data = { + type: Sequelize.INTEGER, + allowNull: true, + defaultValue: null + } + + q.addColumn('Videos', 'language', data, { transaction: utils.transaction }).asCallback(finalCallback) +} + +function down (options, callback) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/initializers/migrator.js b/server/initializers/migrator.js deleted file mode 100644 index 9a6415b1a..000000000 --- a/server/initializers/migrator.js +++ /dev/null @@ -1,139 +0,0 @@ -'use strict' - -const waterfall = require('async/waterfall') -const eachSeries = require('async/eachSeries') -const fs = require('fs') -const path = require('path') - -const constants = require('./constants') -const db = require('./database') -const logger = require('../helpers/logger') - -const migrator = { - migrate: migrate -} - -function migrate (finalCallback) { - waterfall([ - - function checkApplicationTableExists (callback) { - db.sequelize.getQueryInterface().showAllTables().asCallback(function (err, tables) { - if (err) return callback(err) - - // No tables, we don't need to migrate anything - // The installer will do that - if (tables.length === 0) return finalCallback(null) - - return callback(null) - }) - }, - - function loadMigrationVersion (callback) { - db.Application.loadMigrationVersion(callback) - }, - - function createMigrationRowIfNotExists (actualVersion, callback) { - if (actualVersion === null) { - db.Application.create({ - migrationVersion: 0 - }, function (err) { - return callback(err, 0) - }) - } - - return callback(null, actualVersion) - }, - - function abortMigrationIfNotNeeded (actualVersion, callback) { - // No need migrations - if (actualVersion >= constants.LAST_MIGRATION_VERSION) return finalCallback(null) - - return callback(null, actualVersion) - }, - - function getMigrations (actualVersion, callback) { - // If there are a new migration scripts - logger.info('Begin migrations.') - - getMigrationScripts(function (err, migrationScripts) { - return callback(err, actualVersion, migrationScripts) - }) - }, - - function doMigrations (actualVersion, migrationScripts, callback) { - eachSeries(migrationScripts, function (entity, callbackEach) { - executeMigration(actualVersion, entity, callbackEach) - }, function (err) { - if (err) return callback(err) - - logger.info('Migrations finished. New migration version schema: %s', constants.LAST_MIGRATION_VERSION) - return callback(null) - }) - } - ], finalCallback) -} - -// --------------------------------------------------------------------------- - -module.exports = migrator - -// --------------------------------------------------------------------------- - -function getMigrationScripts (callback) { - fs.readdir(path.join(__dirname, 'migrations'), function (err, files) { - if (err) return callback(err) - - const filesToMigrate = [] - - files.forEach(function (file) { - // Filename is something like 'version-blabla.js' - const version = file.split('-')[0] - filesToMigrate.push({ - version, - script: file - }) - }) - - return callback(err, filesToMigrate) - }) -} - -function executeMigration (actualVersion, entity, callback) { - const versionScript = parseInt(entity.version) - - // Do not execute old migration scripts - if (versionScript <= actualVersion) return callback(null) - - // Load the migration module and run it - const migrationScriptName = entity.script - logger.info('Executing %s migration script.', migrationScriptName) - - const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName)) - - db.sequelize.transaction().asCallback(function (err, t) { - if (err) return callback(err) - - const options = { - transaction: t, - queryInterface: db.sequelize.getQueryInterface(), - sequelize: db.sequelize, - Sequelize: db.Sequelize - } - migrationScript.up(options, function (err) { - if (err) { - t.rollback() - return callback(err) - } - - // Update the new migration version - db.Application.updateMigrationVersion(versionScript, t, function (err) { - if (err) { - t.rollback() - return callback(err) - } - - t.commit().asCallback(callback) - }) - }) - }) -} diff --git a/server/initializers/migrator.ts b/server/initializers/migrator.ts new file mode 100644 index 000000000..cfa3220e0 --- /dev/null +++ b/server/initializers/migrator.ts @@ -0,0 +1,134 @@ +import { waterfall, eachSeries } from 'async' +import fs = require('fs') +import path = require('path') + +const db = require('./database') +import { LAST_MIGRATION_VERSION } from './constants' +import { logger } from '../helpers' + +function migrate (finalCallback) { + waterfall([ + + function checkApplicationTableExists (callback) { + db.sequelize.getQueryInterface().showAllTables().asCallback(function (err, tables) { + if (err) return callback(err) + + // No tables, we don't need to migrate anything + // The installer will do that + if (tables.length === 0) return finalCallback(null) + + return callback(null) + }) + }, + + function loadMigrationVersion (callback) { + db.Application.loadMigrationVersion(callback) + }, + + function createMigrationRowIfNotExists (actualVersion, callback) { + if (actualVersion === null) { + db.Application.create({ + migrationVersion: 0 + }, function (err) { + return callback(err, 0) + }) + } + + return callback(null, actualVersion) + }, + + function abortMigrationIfNotNeeded (actualVersion, callback) { + // No need migrations + if (actualVersion >= LAST_MIGRATION_VERSION) return finalCallback(null) + + return callback(null, actualVersion) + }, + + function getMigrations (actualVersion, callback) { + // If there are a new migration scripts + logger.info('Begin migrations.') + + getMigrationScripts(function (err, migrationScripts) { + return callback(err, actualVersion, migrationScripts) + }) + }, + + function doMigrations (actualVersion, migrationScripts, callback) { + eachSeries(migrationScripts, function (entity, callbackEach) { + executeMigration(actualVersion, entity, callbackEach) + }, function (err) { + if (err) return callback(err) + + logger.info('Migrations finished. New migration version schema: %s', LAST_MIGRATION_VERSION) + return callback(null) + }) + } + ], finalCallback) +} + +// --------------------------------------------------------------------------- + +export { + migrate +} + +// --------------------------------------------------------------------------- + +function getMigrationScripts (callback) { + fs.readdir(path.join(__dirname, 'migrations'), function (err, files) { + if (err) return callback(err) + + const filesToMigrate = [] + + files.forEach(function (file) { + // Filename is something like 'version-blabla.js' + const version = file.split('-')[0] + filesToMigrate.push({ + version, + script: file + }) + }) + + return callback(err, filesToMigrate) + }) +} + +function executeMigration (actualVersion, entity, callback) { + const versionScript = parseInt(entity.version) + + // Do not execute old migration scripts + if (versionScript <= actualVersion) return callback(null) + + // Load the migration module and run it + const migrationScriptName = entity.script + logger.info('Executing %s migration script.', migrationScriptName) + + const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName)) + + db.sequelize.transaction().asCallback(function (err, t) { + if (err) return callback(err) + + const options = { + transaction: t, + queryInterface: db.sequelize.getQueryInterface(), + sequelize: db.sequelize, + Sequelize: db.Sequelize + } + migrationScript.up(options, function (err) { + if (err) { + t.rollback() + return callback(err) + } + + // Update the new migration version + db.Application.updateMigrationVersion(versionScript, t, function (err) { + if (err) { + t.rollback() + return callback(err) + } + + t.commit().asCallback(callback) + }) + }) + }) +} diff --git a/server/lib/friends.js b/server/lib/friends.js deleted file mode 100644 index 6dd32406c..000000000 --- a/server/lib/friends.js +++ /dev/null @@ -1,405 +0,0 @@ -'use strict' - -const each = require('async/each') -const eachLimit = require('async/eachLimit') -const eachSeries = require('async/eachSeries') -const series = require('async/series') -const request = require('request') -const waterfall = require('async/waterfall') - -const constants = require('../initializers/constants') -const db = require('../initializers/database') -const logger = require('../helpers/logger') -const peertubeCrypto = require('../helpers/peertube-crypto') -const requests = require('../helpers/requests') -const utils = require('../helpers/utils') -const RequestScheduler = require('./request/request-scheduler') -const RequestVideoQaduScheduler = require('./request/request-video-qadu-scheduler') -const RequestVideoEventScheduler = require('./request/request-video-event-scheduler') - -const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS] - -const requestScheduler = new RequestScheduler() -const requestVideoQaduScheduler = new RequestVideoQaduScheduler() -const requestVideoEventScheduler = new RequestVideoEventScheduler() - -const friends = { - activate, - addVideoToFriends, - updateVideoToFriends, - reportAbuseVideoToFriend, - quickAndDirtyUpdateVideoToFriends, - quickAndDirtyUpdatesVideoToFriends, - addEventToRemoteVideo, - addEventsToRemoteVideo, - hasFriends, - makeFriends, - quitFriends, - removeVideoToFriends, - sendOwnedVideosToPod, - getRequestScheduler, - getRequestVideoQaduScheduler, - getRequestVideoEventScheduler -} - -function activate () { - requestScheduler.activate() - requestVideoQaduScheduler.activate() - requestVideoEventScheduler.activate() -} - -function addVideoToFriends (videoData, transaction, callback) { - const options = { - type: ENDPOINT_ACTIONS.ADD, - endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, - data: videoData, - transaction - } - createRequest(options, callback) -} - -function updateVideoToFriends (videoData, transaction, callback) { - const options = { - type: ENDPOINT_ACTIONS.UPDATE, - endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, - data: videoData, - transaction - } - createRequest(options, callback) -} - -function removeVideoToFriends (videoParams) { - const options = { - type: ENDPOINT_ACTIONS.REMOVE, - endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, - data: videoParams - } - createRequest(options) -} - -function reportAbuseVideoToFriend (reportData, video) { - const options = { - type: ENDPOINT_ACTIONS.REPORT_ABUSE, - endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, - data: reportData, - toIds: [ video.Author.podId ] - } - createRequest(options) -} - -function quickAndDirtyUpdateVideoToFriends (qaduParams, transaction, callback) { - const options = { - videoId: qaduParams.videoId, - type: qaduParams.type, - transaction - } - return createVideoQaduRequest(options, callback) -} - -function quickAndDirtyUpdatesVideoToFriends (qadusParams, transaction, finalCallback) { - const tasks = [] - - qadusParams.forEach(function (qaduParams) { - const fun = function (callback) { - quickAndDirtyUpdateVideoToFriends(qaduParams, transaction, callback) - } - - tasks.push(fun) - }) - - series(tasks, finalCallback) -} - -function addEventToRemoteVideo (eventParams, transaction, callback) { - const options = { - videoId: eventParams.videoId, - type: eventParams.type, - transaction - } - createVideoEventRequest(options, callback) -} - -function addEventsToRemoteVideo (eventsParams, transaction, finalCallback) { - const tasks = [] - - eventsParams.forEach(function (eventParams) { - const fun = function (callback) { - addEventToRemoteVideo(eventParams, transaction, callback) - } - - tasks.push(fun) - }) - - series(tasks, finalCallback) -} - -function hasFriends (callback) { - db.Pod.countAll(function (err, count) { - if (err) return callback(err) - - const hasFriends = (count !== 0) - callback(null, hasFriends) - }) -} - -function makeFriends (hosts, callback) { - const podsScore = {} - - logger.info('Make friends!') - peertubeCrypto.getMyPublicCert(function (err, cert) { - if (err) { - logger.error('Cannot read public cert.') - return callback(err) - } - - eachSeries(hosts, function (host, callbackEach) { - computeForeignPodsList(host, podsScore, callbackEach) - }, function (err) { - if (err) return callback(err) - - logger.debug('Pods scores computed.', { podsScore: podsScore }) - const podsList = computeWinningPods(hosts, podsScore) - logger.debug('Pods that we keep.', { podsToKeep: podsList }) - - makeRequestsToWinningPods(cert, podsList, callback) - }) - }) -} - -function quitFriends (callback) { - // Stop pool requests - requestScheduler.deactivate() - - waterfall([ - function flushRequests (callbackAsync) { - requestScheduler.flush(err => callbackAsync(err)) - }, - - function flushVideoQaduRequests (callbackAsync) { - requestVideoQaduScheduler.flush(err => callbackAsync(err)) - }, - - function getPodsList (callbackAsync) { - return db.Pod.list(callbackAsync) - }, - - function announceIQuitMyFriends (pods, callbackAsync) { - const requestParams = { - method: 'POST', - path: '/api/' + constants.API_VERSION + '/remote/pods/remove', - sign: true - } - - // Announce we quit them - // We don't care if the request fails - // The other pod will exclude us automatically after a while - eachLimit(pods, constants.REQUESTS_IN_PARALLEL, function (pod, callbackEach) { - requestParams.toPod = pod - requests.makeSecureRequest(requestParams, callbackEach) - }, function (err) { - if (err) { - logger.error('Some errors while quitting friends.', { err: err }) - // Don't stop the process - } - - return callbackAsync(null, pods) - }) - }, - - function removePodsFromDB (pods, callbackAsync) { - each(pods, function (pod, callbackEach) { - pod.destroy().asCallback(callbackEach) - }, callbackAsync) - } - ], function (err) { - // Don't forget to re activate the scheduler, even if there was an error - requestScheduler.activate() - - if (err) return callback(err) - - logger.info('Removed all remote videos.') - return callback(null) - }) -} - -function sendOwnedVideosToPod (podId) { - db.Video.listOwnedAndPopulateAuthorAndTags(function (err, videosList) { - if (err) { - logger.error('Cannot get the list of videos we own.') - return - } - - videosList.forEach(function (video) { - video.toAddRemoteJSON(function (err, remoteVideo) { - if (err) { - logger.error('Cannot convert video to remote.', { error: err }) - // Don't break the process - return - } - - const options = { - type: 'add', - endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, - data: remoteVideo, - toIds: [ podId ] - } - createRequest(options) - }) - }) - }) -} - -function getRequestScheduler () { - return requestScheduler -} - -function getRequestVideoQaduScheduler () { - return requestVideoQaduScheduler -} - -function getRequestVideoEventScheduler () { - return requestVideoEventScheduler -} - -// --------------------------------------------------------------------------- - -module.exports = friends - -// --------------------------------------------------------------------------- - -function computeForeignPodsList (host, podsScore, callback) { - getForeignPodsList(host, function (err, res) { - if (err) return callback(err) - - const foreignPodsList = res.data - - // Let's give 1 point to the pod we ask the friends list - foreignPodsList.push({ host }) - - foreignPodsList.forEach(function (foreignPod) { - const foreignPodHost = foreignPod.host - - if (podsScore[foreignPodHost]) podsScore[foreignPodHost]++ - else podsScore[foreignPodHost] = 1 - }) - - return callback() - }) -} - -function computeWinningPods (hosts, podsScore) { - // Build the list of pods to add - // Only add a pod if it exists in more than a half base pods - const podsList = [] - const baseScore = hosts.length / 2 - - Object.keys(podsScore).forEach(function (podHost) { - // If the pod is not me and with a good score we add it - if (isMe(podHost) === false && podsScore[podHost] > baseScore) { - podsList.push({ host: podHost }) - } - }) - - return podsList -} - -function getForeignPodsList (host, callback) { - const path = '/api/' + constants.API_VERSION + '/pods' - - request.get(constants.REMOTE_SCHEME.HTTP + '://' + host + path, function (err, response, body) { - if (err) return callback(err) - - try { - const json = JSON.parse(body) - return callback(null, json) - } catch (err) { - return callback(err) - } - }) -} - -function makeRequestsToWinningPods (cert, podsList, callback) { - // Stop pool requests - requestScheduler.deactivate() - // Flush pool requests - requestScheduler.forceSend() - - eachLimit(podsList, constants.REQUESTS_IN_PARALLEL, function (pod, callbackEach) { - const params = { - url: constants.REMOTE_SCHEME.HTTP + '://' + pod.host + '/api/' + constants.API_VERSION + '/pods/', - method: 'POST', - json: { - host: constants.CONFIG.WEBSERVER.HOST, - email: constants.CONFIG.ADMIN.EMAIL, - publicKey: cert - } - } - - requests.makeRetryRequest(params, function (err, res, body) { - if (err) { - logger.error('Error with adding %s pod.', pod.host, { error: err }) - // Don't break the process - return callbackEach() - } - - if (res.statusCode === 200) { - const podObj = db.Pod.build({ host: pod.host, publicKey: body.cert, email: body.email }) - podObj.save().asCallback(function (err, podCreated) { - if (err) { - logger.error('Cannot add friend %s pod.', pod.host, { error: err }) - return callbackEach() - } - - // Add our videos to the request scheduler - sendOwnedVideosToPod(podCreated.id) - - return callbackEach() - }) - } else { - logger.error('Status not 200 for %s pod.', pod.host) - return callbackEach() - } - }) - }, function endRequests () { - // Final callback, we've ended all the requests - // Now we made new friends, we can re activate the pool of requests - requestScheduler.activate() - - logger.debug('makeRequestsToWinningPods finished.') - return callback() - }) -} - -// Wrapper that populate "toIds" argument with all our friends if it is not specified -// { type, endpoint, data, toIds, transaction } -function createRequest (options, callback) { - if (!callback) callback = function () {} - if (options.toIds) return requestScheduler.createRequest(options, callback) - - // If the "toIds" pods is not specified, we send the request to all our friends - db.Pod.listAllIds(options.transaction, function (err, podIds) { - if (err) { - logger.error('Cannot get pod ids', { error: err }) - return - } - - const newOptions = Object.assign(options, { toIds: podIds }) - return requestScheduler.createRequest(newOptions, callback) - }) -} - -function createVideoQaduRequest (options, callback) { - if (!callback) callback = utils.createEmptyCallback() - - requestVideoQaduScheduler.createRequest(options, callback) -} - -function createVideoEventRequest (options, callback) { - if (!callback) callback = utils.createEmptyCallback() - - requestVideoEventScheduler.createRequest(options, callback) -} - -function isMe (host) { - return host === constants.CONFIG.WEBSERVER.HOST -} diff --git a/server/lib/friends.ts b/server/lib/friends.ts new file mode 100644 index 000000000..b32783019 --- /dev/null +++ b/server/lib/friends.ts @@ -0,0 +1,410 @@ +import { each, eachLimit, eachSeries, series, waterfall } from 'async' +import request = require('request') + +const db = require('../initializers/database') +import { + API_VERSION, + CONFIG, + REQUESTS_IN_PARALLEL, + REQUEST_ENDPOINTS, + REQUEST_ENDPOINT_ACTIONS, + REMOTE_SCHEME +} from '../initializers' +import { + logger, + getMyPublicCert, + makeSecureRequest, + makeRetryRequest, + createEmptyCallback +} from '../helpers' +import { + RequestScheduler, + RequestVideoQaduScheduler, + RequestVideoEventScheduler +} from './request' + +const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] + +const requestScheduler = new RequestScheduler() +const requestVideoQaduScheduler = new RequestVideoQaduScheduler() +const requestVideoEventScheduler = new RequestVideoEventScheduler() + +function activateSchedulers () { + requestScheduler.activate() + requestVideoQaduScheduler.activate() + requestVideoEventScheduler.activate() +} + +function addVideoToFriends (videoData, transaction, callback) { + const options = { + type: ENDPOINT_ACTIONS.ADD, + endpoint: REQUEST_ENDPOINTS.VIDEOS, + data: videoData, + transaction + } + createRequest(options, callback) +} + +function updateVideoToFriends (videoData, transaction, callback) { + const options = { + type: ENDPOINT_ACTIONS.UPDATE, + endpoint: REQUEST_ENDPOINTS.VIDEOS, + data: videoData, + transaction + } + createRequest(options, callback) +} + +function removeVideoToFriends (videoParams) { + const options = { + type: ENDPOINT_ACTIONS.REMOVE, + endpoint: REQUEST_ENDPOINTS.VIDEOS, + data: videoParams + } + createRequest(options) +} + +function reportAbuseVideoToFriend (reportData, video) { + const options = { + type: ENDPOINT_ACTIONS.REPORT_ABUSE, + endpoint: REQUEST_ENDPOINTS.VIDEOS, + data: reportData, + toIds: [ video.Author.podId ] + } + createRequest(options) +} + +function quickAndDirtyUpdateVideoToFriends (qaduParams, transaction?, callback?) { + const options = { + videoId: qaduParams.videoId, + type: qaduParams.type, + transaction + } + return createVideoQaduRequest(options, callback) +} + +function quickAndDirtyUpdatesVideoToFriends (qadusParams, transaction, finalCallback) { + const tasks = [] + + qadusParams.forEach(function (qaduParams) { + const fun = function (callback) { + quickAndDirtyUpdateVideoToFriends(qaduParams, transaction, callback) + } + + tasks.push(fun) + }) + + series(tasks, finalCallback) +} + +function addEventToRemoteVideo (eventParams, transaction?, callback?) { + const options = { + videoId: eventParams.videoId, + type: eventParams.type, + transaction + } + createVideoEventRequest(options, callback) +} + +function addEventsToRemoteVideo (eventsParams, transaction, finalCallback) { + const tasks = [] + + eventsParams.forEach(function (eventParams) { + const fun = function (callback) { + addEventToRemoteVideo(eventParams, transaction, callback) + } + + tasks.push(fun) + }) + + series(tasks, finalCallback) +} + +function hasFriends (callback) { + db.Pod.countAll(function (err, count) { + if (err) return callback(err) + + const hasFriends = (count !== 0) + callback(null, hasFriends) + }) +} + +function makeFriends (hosts, callback) { + const podsScore = {} + + logger.info('Make friends!') + getMyPublicCert(function (err, cert) { + if (err) { + logger.error('Cannot read public cert.') + return callback(err) + } + + eachSeries(hosts, function (host, callbackEach) { + computeForeignPodsList(host, podsScore, callbackEach) + }, function (err) { + if (err) return callback(err) + + logger.debug('Pods scores computed.', { podsScore: podsScore }) + const podsList = computeWinningPods(hosts, podsScore) + logger.debug('Pods that we keep.', { podsToKeep: podsList }) + + makeRequestsToWinningPods(cert, podsList, callback) + }) + }) +} + +function quitFriends (callback) { + // Stop pool requests + requestScheduler.deactivate() + + waterfall([ + function flushRequests (callbackAsync) { + requestScheduler.flush(err => callbackAsync(err)) + }, + + function flushVideoQaduRequests (callbackAsync) { + requestVideoQaduScheduler.flush(err => callbackAsync(err)) + }, + + function getPodsList (callbackAsync) { + return db.Pod.list(callbackAsync) + }, + + function announceIQuitMyFriends (pods, callbackAsync) { + const requestParams = { + method: 'POST', + path: '/api/' + API_VERSION + '/remote/pods/remove', + sign: true, + toPod: null + } + + // Announce we quit them + // We don't care if the request fails + // The other pod will exclude us automatically after a while + eachLimit(pods, REQUESTS_IN_PARALLEL, function (pod, callbackEach) { + requestParams.toPod = pod + makeSecureRequest(requestParams, callbackEach) + }, function (err) { + if (err) { + logger.error('Some errors while quitting friends.', { err: err }) + // Don't stop the process + } + + return callbackAsync(null, pods) + }) + }, + + function removePodsFromDB (pods, callbackAsync) { + each(pods, function (pod: any, callbackEach) { + pod.destroy().asCallback(callbackEach) + }, callbackAsync) + } + ], function (err) { + // Don't forget to re activate the scheduler, even if there was an error + requestScheduler.activate() + + if (err) return callback(err) + + logger.info('Removed all remote videos.') + return callback(null) + }) +} + +function sendOwnedVideosToPod (podId) { + db.Video.listOwnedAndPopulateAuthorAndTags(function (err, videosList) { + if (err) { + logger.error('Cannot get the list of videos we own.') + return + } + + videosList.forEach(function (video) { + video.toAddRemoteJSON(function (err, remoteVideo) { + if (err) { + logger.error('Cannot convert video to remote.', { error: err }) + // Don't break the process + return + } + + const options = { + type: 'add', + endpoint: REQUEST_ENDPOINTS.VIDEOS, + data: remoteVideo, + toIds: [ podId ] + } + createRequest(options) + }) + }) + }) +} + +function getRequestScheduler () { + return requestScheduler +} + +function getRequestVideoQaduScheduler () { + return requestVideoQaduScheduler +} + +function getRequestVideoEventScheduler () { + return requestVideoEventScheduler +} + +// --------------------------------------------------------------------------- + +export { + activateSchedulers, + addVideoToFriends, + updateVideoToFriends, + reportAbuseVideoToFriend, + quickAndDirtyUpdateVideoToFriends, + quickAndDirtyUpdatesVideoToFriends, + addEventToRemoteVideo, + addEventsToRemoteVideo, + hasFriends, + makeFriends, + quitFriends, + removeVideoToFriends, + sendOwnedVideosToPod, + getRequestScheduler, + getRequestVideoQaduScheduler, + getRequestVideoEventScheduler +} + +// --------------------------------------------------------------------------- + +function computeForeignPodsList (host, podsScore, callback) { + getForeignPodsList(host, function (err, res) { + if (err) return callback(err) + + const foreignPodsList = res.data + + // Let's give 1 point to the pod we ask the friends list + foreignPodsList.push({ host }) + + foreignPodsList.forEach(function (foreignPod) { + const foreignPodHost = foreignPod.host + + if (podsScore[foreignPodHost]) podsScore[foreignPodHost]++ + else podsScore[foreignPodHost] = 1 + }) + + return callback() + }) +} + +function computeWinningPods (hosts, podsScore) { + // Build the list of pods to add + // Only add a pod if it exists in more than a half base pods + const podsList = [] + const baseScore = hosts.length / 2 + + Object.keys(podsScore).forEach(function (podHost) { + // If the pod is not me and with a good score we add it + if (isMe(podHost) === false && podsScore[podHost] > baseScore) { + podsList.push({ host: podHost }) + } + }) + + return podsList +} + +function getForeignPodsList (host, callback) { + const path = '/api/' + API_VERSION + '/pods' + + request.get(REMOTE_SCHEME.HTTP + '://' + host + path, function (err, response, body) { + if (err) return callback(err) + + try { + const json = JSON.parse(body) + return callback(null, json) + } catch (err) { + return callback(err) + } + }) +} + +function makeRequestsToWinningPods (cert, podsList, callback) { + // Stop pool requests + requestScheduler.deactivate() + // Flush pool requests + requestScheduler.forceSend() + + eachLimit(podsList, REQUESTS_IN_PARALLEL, function (pod: any, callbackEach) { + const params = { + url: REMOTE_SCHEME.HTTP + '://' + pod.host + '/api/' + API_VERSION + '/pods/', + method: 'POST', + json: { + host: CONFIG.WEBSERVER.HOST, + email: CONFIG.ADMIN.EMAIL, + publicKey: cert + } + } + + makeRetryRequest(params, function (err, res, body) { + if (err) { + logger.error('Error with adding %s pod.', pod.host, { error: err }) + // Don't break the process + return callbackEach() + } + + if (res.statusCode === 200) { + const podObj = db.Pod.build({ host: pod.host, publicKey: body.cert, email: body.email }) + podObj.save().asCallback(function (err, podCreated) { + if (err) { + logger.error('Cannot add friend %s pod.', pod.host, { error: err }) + return callbackEach() + } + + // Add our videos to the request scheduler + sendOwnedVideosToPod(podCreated.id) + + return callbackEach() + }) + } else { + logger.error('Status not 200 for %s pod.', pod.host) + return callbackEach() + } + }) + }, function endRequests () { + // Final callback, we've ended all the requests + // Now we made new friends, we can re activate the pool of requests + requestScheduler.activate() + + logger.debug('makeRequestsToWinningPods finished.') + return callback() + }) +} + +// Wrapper that populate "toIds" argument with all our friends if it is not specified +// { type, endpoint, data, toIds, transaction } +function createRequest (options, callback?) { + if (!callback) callback = function () { /* empty */ } + if (options.toIds) return requestScheduler.createRequest(options, callback) + + // If the "toIds" pods is not specified, we send the request to all our friends + db.Pod.listAllIds(options.transaction, function (err, podIds) { + if (err) { + logger.error('Cannot get pod ids', { error: err }) + return + } + + const newOptions = Object.assign(options, { toIds: podIds }) + return requestScheduler.createRequest(newOptions, callback) + }) +} + +function createVideoQaduRequest (options, callback) { + if (!callback) callback = createEmptyCallback() + + requestVideoQaduScheduler.createRequest(options, callback) +} + +function createVideoEventRequest (options, callback) { + if (!callback) callback = createEmptyCallback() + + requestVideoEventScheduler.createRequest(options, callback) +} + +function isMe (host) { + return host === CONFIG.WEBSERVER.HOST +} diff --git a/server/lib/index.ts b/server/lib/index.ts new file mode 100644 index 000000000..b8697fb96 --- /dev/null +++ b/server/lib/index.ts @@ -0,0 +1,4 @@ +export * from './jobs' +export * from './request' +export * from './friends' +export * from './oauth-model' diff --git a/server/lib/jobs/handlers/index.js b/server/lib/jobs/handlers/index.js deleted file mode 100644 index 59c1ccce5..000000000 --- a/server/lib/jobs/handlers/index.js +++ /dev/null @@ -1,7 +0,0 @@ -'use strict' - -const videoTranscoder = require('./video-transcoder') - -module.exports = { - videoTranscoder -} diff --git a/server/lib/jobs/handlers/index.ts b/server/lib/jobs/handlers/index.ts new file mode 100644 index 000000000..ae5440031 --- /dev/null +++ b/server/lib/jobs/handlers/index.ts @@ -0,0 +1,9 @@ +import * as videoTranscoder from './video-transcoder' + +const jobHandlers = { + videoTranscoder +} + +export { + jobHandlers +} diff --git a/server/lib/jobs/handlers/video-transcoder.js b/server/lib/jobs/handlers/video-transcoder.js deleted file mode 100644 index d2ad4f9c7..000000000 --- a/server/lib/jobs/handlers/video-transcoder.js +++ /dev/null @@ -1,43 +0,0 @@ -'use strict' - -const db = require('../../../initializers/database') -const logger = require('../../../helpers/logger') -const friends = require('../../../lib/friends') - -const VideoTranscoderHandler = { - process, - onError, - onSuccess -} - -// --------------------------------------------------------------------------- - -function process (data, callback) { - db.Video.loadAndPopulateAuthorAndPodAndTags(data.id, function (err, video) { - if (err) return callback(err) - - video.transcodeVideofile(function (err) { - return callback(err, video) - }) - }) -} - -function onError (err, jobId, video, callback) { - logger.error('Error when transcoding video file in job %d.', jobId, { error: err }) - return callback() -} - -function onSuccess (data, jobId, video, callback) { - logger.info('Job %d is a success.', jobId) - - video.toAddRemoteJSON(function (err, remoteVideo) { - if (err) return callback(err) - - // Now we'll add the video's meta data to our friends - friends.addVideoToFriends(remoteVideo, null, callback) - }) -} - -// --------------------------------------------------------------------------- - -module.exports = VideoTranscoderHandler diff --git a/server/lib/jobs/handlers/video-transcoder.ts b/server/lib/jobs/handlers/video-transcoder.ts new file mode 100644 index 000000000..35db5fb96 --- /dev/null +++ b/server/lib/jobs/handlers/video-transcoder.ts @@ -0,0 +1,37 @@ +const db = require('../../../initializers/database') +import { logger } from '../../../helpers' +import { addVideoToFriends } from '../../../lib' + +function process (data, callback) { + db.Video.loadAndPopulateAuthorAndPodAndTags(data.id, function (err, video) { + if (err) return callback(err) + + video.transcodeVideofile(function (err) { + return callback(err, video) + }) + }) +} + +function onError (err, jobId, video, callback) { + logger.error('Error when transcoding video file in job %d.', jobId, { error: err }) + return callback() +} + +function onSuccess (data, jobId, video, callback) { + logger.info('Job %d is a success.', jobId) + + video.toAddRemoteJSON(function (err, remoteVideo) { + if (err) return callback(err) + + // Now we'll add the video's meta data to our friends + addVideoToFriends(remoteVideo, null, callback) + }) +} + +// --------------------------------------------------------------------------- + +export { + process, + onError, + onSuccess +} diff --git a/server/lib/jobs/index.ts b/server/lib/jobs/index.ts new file mode 100644 index 000000000..b18a3d845 --- /dev/null +++ b/server/lib/jobs/index.ts @@ -0,0 +1 @@ +export * from './job-scheduler' diff --git a/server/lib/jobs/job-scheduler.js b/server/lib/jobs/job-scheduler.js deleted file mode 100644 index 7b239577f..000000000 --- a/server/lib/jobs/job-scheduler.js +++ /dev/null @@ -1,129 +0,0 @@ -'use strict' - -const forever = require('async/forever') -const queue = require('async/queue') - -const constants = require('../../initializers/constants') -const db = require('../../initializers/database') -const logger = require('../../helpers/logger') - -const jobHandlers = require('./handlers') - -const jobScheduler = { - activate, - createJob -} - -function activate () { - const limit = constants.JOBS_FETCH_LIMIT_PER_CYCLE - - logger.info('Jobs scheduler activated.') - - const jobsQueue = queue(processJob) - - // Finish processing jobs from a previous start - const state = constants.JOB_STATES.PROCESSING - db.Job.listWithLimit(limit, state, function (err, jobs) { - enqueueJobs(err, jobsQueue, jobs) - - forever( - function (next) { - if (jobsQueue.length() !== 0) { - // Finish processing the queue first - return setTimeout(next, constants.JOBS_FETCHING_INTERVAL) - } - - const state = constants.JOB_STATES.PENDING - db.Job.listWithLimit(limit, state, function (err, jobs) { - if (err) { - logger.error('Cannot list pending jobs.', { error: err }) - } else { - jobs.forEach(function (job) { - jobsQueue.push(job) - }) - } - - // Optimization: we could use "drain" from queue object - return setTimeout(next, constants.JOBS_FETCHING_INTERVAL) - }) - } - ) - }) -} - -// --------------------------------------------------------------------------- - -module.exports = jobScheduler - -// --------------------------------------------------------------------------- - -function enqueueJobs (err, jobsQueue, jobs) { - if (err) { - logger.error('Cannot list pending jobs.', { error: err }) - } else { - jobs.forEach(function (job) { - jobsQueue.push(job) - }) - } -} - -function createJob (transaction, handlerName, handlerInputData, callback) { - const createQuery = { - state: constants.JOB_STATES.PENDING, - handlerName, - handlerInputData - } - const options = { transaction } - - db.Job.create(createQuery, options).asCallback(callback) -} - -function processJob (job, callback) { - const jobHandler = jobHandlers[job.handlerName] - - logger.info('Processing job %d with handler %s.', job.id, job.handlerName) - - job.state = constants.JOB_STATES.PROCESSING - job.save().asCallback(function (err) { - if (err) return cannotSaveJobError(err, callback) - - if (jobHandler === undefined) { - logger.error('Unknown job handler for job %s.', jobHandler.handlerName) - return callback() - } - - return jobHandler.process(job.handlerInputData, function (err, result) { - if (err) { - logger.error('Error in job handler %s.', job.handlerName, { error: err }) - return onJobError(jobHandler, job, result, callback) - } - - return onJobSuccess(jobHandler, job, result, callback) - }) - }) -} - -function onJobError (jobHandler, job, jobResult, callback) { - job.state = constants.JOB_STATES.ERROR - - job.save().asCallback(function (err) { - if (err) return cannotSaveJobError(err, callback) - - return jobHandler.onError(err, job.id, jobResult, callback) - }) -} - -function onJobSuccess (jobHandler, job, jobResult, callback) { - job.state = constants.JOB_STATES.SUCCESS - - job.save().asCallback(function (err) { - if (err) return cannotSaveJobError(err, callback) - - return jobHandler.onSuccess(err, job.id, jobResult, callback) - }) -} - -function cannotSaveJobError (err, callback) { - logger.error('Cannot save new job state.', { error: err }) - return callback(err) -} diff --git a/server/lib/jobs/job-scheduler.ts b/server/lib/jobs/job-scheduler.ts new file mode 100644 index 000000000..7b8c6faf9 --- /dev/null +++ b/server/lib/jobs/job-scheduler.ts @@ -0,0 +1,137 @@ +import { forever, queue } from 'async' + +const db = require('../../initializers/database') +import { + JOBS_FETCHING_INTERVAL, + JOBS_FETCH_LIMIT_PER_CYCLE, + JOB_STATES +} from '../../initializers' +import { logger } from '../../helpers' +import { jobHandlers } from './handlers' + +class JobScheduler { + + private static instance: JobScheduler + + private constructor () { } + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + activate () { + const limit = JOBS_FETCH_LIMIT_PER_CYCLE + + logger.info('Jobs scheduler activated.') + + const jobsQueue = queue(this.processJob) + + // Finish processing jobs from a previous start + const state = JOB_STATES.PROCESSING + db.Job.listWithLimit(limit, state, (err, jobs) => { + this.enqueueJobs(err, jobsQueue, jobs) + + forever( + next => { + if (jobsQueue.length() !== 0) { + // Finish processing the queue first + return setTimeout(next, JOBS_FETCHING_INTERVAL) + } + + const state = JOB_STATES.PENDING + db.Job.listWithLimit(limit, state, (err, jobs) => { + if (err) { + logger.error('Cannot list pending jobs.', { error: err }) + } else { + jobs.forEach(job => { + jobsQueue.push(job) + }) + } + + // Optimization: we could use "drain" from queue object + return setTimeout(next, JOBS_FETCHING_INTERVAL) + }) + }, + + err => { logger.error('Error in job scheduler queue.', { error: err }) } + ) + }) + } + + createJob (transaction, handlerName, handlerInputData, callback) { + const createQuery = { + state: JOB_STATES.PENDING, + handlerName, + handlerInputData + } + const options = { transaction } + + db.Job.create(createQuery, options).asCallback(callback) + } + + private enqueueJobs (err, jobsQueue, jobs) { + if (err) { + logger.error('Cannot list pending jobs.', { error: err }) + } else { + jobs.forEach(job => { + jobsQueue.push(job) + }) + } + } + + private processJob (job, callback) { + const jobHandler = jobHandlers[job.handlerName] + + logger.info('Processing job %d with handler %s.', job.id, job.handlerName) + + job.state = JOB_STATES.PROCESSING + job.save().asCallback(err => { + if (err) return this.cannotSaveJobError(err, callback) + + if (jobHandler === undefined) { + logger.error('Unknown job handler for job %s.', jobHandler.handlerName) + return callback() + } + + return jobHandler.process(job.handlerInputData, (err, result) => { + if (err) { + logger.error('Error in job handler %s.', job.handlerName, { error: err }) + return this.onJobError(jobHandler, job, result, callback) + } + + return this.onJobSuccess(jobHandler, job, result, callback) + }) + }) + } + + private onJobError (jobHandler, job, jobResult, callback) { + job.state = JOB_STATES.ERROR + + job.save().asCallback(err => { + if (err) return this.cannotSaveJobError(err, callback) + + return jobHandler.onError(err, job.id, jobResult, callback) + }) + } + + private onJobSuccess (jobHandler, job, jobResult, callback) { + job.state = JOB_STATES.SUCCESS + + job.save().asCallback(err => { + if (err) return this.cannotSaveJobError(err, callback) + + return jobHandler.onSuccess(err, job.id, jobResult, callback) + }) + } + + private cannotSaveJobError (err, callback) { + logger.error('Cannot save new job state.', { error: err }) + return callback(err) + } +} + +// --------------------------------------------------------------------------- + +export { + JobScheduler +} diff --git a/server/lib/oauth-model.js b/server/lib/oauth-model.js deleted file mode 100644 index 1c12f1b14..000000000 --- a/server/lib/oauth-model.js +++ /dev/null @@ -1,97 +0,0 @@ -const db = require('../initializers/database') -const logger = require('../helpers/logger') - -// See https://github.com/oauthjs/node-oauth2-server/wiki/Model-specification for the model specifications -const OAuthModel = { - getAccessToken, - getClient, - getRefreshToken, - getUser, - revokeToken, - saveToken -} - -// --------------------------------------------------------------------------- - -function getAccessToken (bearerToken) { - logger.debug('Getting access token (bearerToken: ' + bearerToken + ').') - - return db.OAuthToken.getByTokenAndPopulateUser(bearerToken) -} - -function getClient (clientId, clientSecret) { - logger.debug('Getting Client (clientId: ' + clientId + ', clientSecret: ' + clientSecret + ').') - - return db.OAuthClient.getByIdAndSecret(clientId, clientSecret) -} - -function getRefreshToken (refreshToken) { - logger.debug('Getting RefreshToken (refreshToken: ' + refreshToken + ').') - - return db.OAuthToken.getByRefreshTokenAndPopulateClient(refreshToken) -} - -function getUser (username, password) { - logger.debug('Getting User (username: ' + username + ', password: ' + password + ').') - - return db.User.getByUsername(username).then(function (user) { - if (!user) return null - - // We need to return a promise - return new Promise(function (resolve, reject) { - return user.isPasswordMatch(password, function (err, isPasswordMatch) { - if (err) return reject(err) - - if (isPasswordMatch === true) { - return resolve(user) - } - - return resolve(null) - }) - }) - }) -} - -function revokeToken (token) { - return db.OAuthToken.getByRefreshTokenAndPopulateUser(token.refreshToken).then(function (tokenDB) { - if (tokenDB) tokenDB.destroy() - - /* - * Thanks to https://github.com/manjeshpv/node-oauth2-server-implementation/blob/master/components/oauth/mongo-models.js - * "As per the discussion we need set older date - * revokeToken will expected return a boolean in future version - * https://github.com/oauthjs/node-oauth2-server/pull/274 - * https://github.com/oauthjs/node-oauth2-server/issues/290" - */ - const expiredToken = tokenDB - expiredToken.refreshTokenExpiresAt = new Date('2015-05-28T06:59:53.000Z') - - return expiredToken - }) -} - -function saveToken (token, client, user) { - logger.debug('Saving token ' + token.accessToken + ' for client ' + client.id + ' and user ' + user.id + '.') - - const tokenToCreate = { - accessToken: token.accessToken, - accessTokenExpiresAt: token.accessTokenExpiresAt, - refreshToken: token.refreshToken, - refreshTokenExpiresAt: token.refreshTokenExpiresAt, - oAuthClientId: client.id, - userId: user.id - } - - return db.OAuthToken.create(tokenToCreate).then(function (tokenCreated) { - tokenCreated.client = client - tokenCreated.user = user - - return tokenCreated - }).catch(function (err) { - throw err - }) -} - -// --------------------------------------------------------------------------- - -module.exports = OAuthModel diff --git a/server/lib/oauth-model.ts b/server/lib/oauth-model.ts new file mode 100644 index 000000000..00b1afcf5 --- /dev/null +++ b/server/lib/oauth-model.ts @@ -0,0 +1,95 @@ +const db = require('../initializers/database') +import { logger } from '../helpers' + +// --------------------------------------------------------------------------- + +function getAccessToken (bearerToken) { + logger.debug('Getting access token (bearerToken: ' + bearerToken + ').') + + return db.OAuthToken.getByTokenAndPopulateUser(bearerToken) +} + +function getClient (clientId, clientSecret) { + logger.debug('Getting Client (clientId: ' + clientId + ', clientSecret: ' + clientSecret + ').') + + return db.OAuthClient.getByIdAndSecret(clientId, clientSecret) +} + +function getRefreshToken (refreshToken) { + logger.debug('Getting RefreshToken (refreshToken: ' + refreshToken + ').') + + return db.OAuthToken.getByRefreshTokenAndPopulateClient(refreshToken) +} + +function getUser (username, password) { + logger.debug('Getting User (username: ' + username + ', password: ' + password + ').') + + return db.User.getByUsername(username).then(function (user) { + if (!user) return null + + // We need to return a promise + return new Promise(function (resolve, reject) { + return user.isPasswordMatch(password, function (err, isPasswordMatch) { + if (err) return reject(err) + + if (isPasswordMatch === true) { + return resolve(user) + } + + return resolve(null) + }) + }) + }) +} + +function revokeToken (token) { + return db.OAuthToken.getByRefreshTokenAndPopulateUser(token.refreshToken).then(function (tokenDB) { + if (tokenDB) tokenDB.destroy() + + /* + * Thanks to https://github.com/manjeshpv/node-oauth2-server-implementation/blob/master/components/oauth/mongo-models.js + * "As per the discussion we need set older date + * revokeToken will expected return a boolean in future version + * https://github.com/oauthjs/node-oauth2-server/pull/274 + * https://github.com/oauthjs/node-oauth2-server/issues/290" + */ + const expiredToken = tokenDB + expiredToken.refreshTokenExpiresAt = new Date('2015-05-28T06:59:53.000Z') + + return expiredToken + }) +} + +function saveToken (token, client, user) { + logger.debug('Saving token ' + token.accessToken + ' for client ' + client.id + ' and user ' + user.id + '.') + + const tokenToCreate = { + accessToken: token.accessToken, + accessTokenExpiresAt: token.accessTokenExpiresAt, + refreshToken: token.refreshToken, + refreshTokenExpiresAt: token.refreshTokenExpiresAt, + oAuthClientId: client.id, + userId: user.id + } + + return db.OAuthToken.create(tokenToCreate).then(function (tokenCreated) { + tokenCreated.client = client + tokenCreated.user = user + + return tokenCreated + }).catch(function (err) { + throw err + }) +} + +// --------------------------------------------------------------------------- + +// See https://github.com/oauthjs/node-oauth2-server/wiki/Model-specification for the model specifications +export { + getAccessToken, + getClient, + getRefreshToken, + getUser, + revokeToken, + saveToken +} diff --git a/server/lib/request/base-request-scheduler.js b/server/lib/request/base-request-scheduler.js deleted file mode 100644 index 782448340..000000000 --- a/server/lib/request/base-request-scheduler.js +++ /dev/null @@ -1,136 +0,0 @@ -'use strict' - -const eachLimit = require('async/eachLimit') - -const constants = require('../../initializers/constants') -const db = require('../../initializers/database') -const logger = require('../../helpers/logger') -const requests = require('../../helpers/requests') - -module.exports = class BaseRequestScheduler { - constructor (options) { - this.lastRequestTimestamp = 0 - this.timer = null - this.requestInterval = constants.REQUESTS_INTERVAL - } - - activate () { - logger.info('Requests scheduler activated.') - this.lastRequestTimestamp = Date.now() - - this.timer = setInterval(() => { - this.lastRequestTimestamp = Date.now() - this.makeRequests() - }, this.requestInterval) - } - - deactivate () { - logger.info('Requests scheduler deactivated.') - clearInterval(this.timer) - this.timer = null - } - - forceSend () { - logger.info('Force requests scheduler sending.') - this.makeRequests() - } - - remainingMilliSeconds () { - if (this.timer === null) return -1 - - return constants.REQUESTS_INTERVAL - (Date.now() - this.lastRequestTimestamp) - } - - remainingRequestsCount (callback) { - return this.getRequestModel().countTotalRequests(callback) - } - - // --------------------------------------------------------------------------- - - // Make a requests to friends of a certain type - makeRequest (toPod, requestEndpoint, requestsToMake, callback) { - if (!callback) callback = function () {} - - const params = { - toPod: toPod, - sign: true, // Prove our identity - method: 'POST', - path: '/api/' + constants.API_VERSION + '/remote/' + requestEndpoint, - data: requestsToMake // Requests we need to make - } - - // Make multiple retry requests to all of pods - // The function fire some useful callbacks - requests.makeSecureRequest(params, (err, res) => { - if (err || (res.statusCode !== 200 && res.statusCode !== 201 && res.statusCode !== 204)) { - err = err ? err.message : 'Status code not 20x : ' + res.statusCode - logger.error('Error sending secure request to %s pod.', toPod.host, { error: err }) - - return callback(err) - } - - return callback(null) - }) - } - - // Make all the requests of the scheduler - makeRequests () { - this.getRequestModel().listWithLimitAndRandom(this.limitPods, this.limitPerPod, (err, requests) => { - if (err) { - logger.error('Cannot get the list of "%s".', this.description, { err: err }) - return // Abort - } - - // If there are no requests, abort - if (requests.length === 0) { - logger.info('No "%s" to make.', this.description) - return - } - - // We want to group requests by destinations pod and endpoint - const requestsToMakeGrouped = this.buildRequestObjects(requests) - - logger.info('Making "%s" to friends.', this.description) - - const goodPods = [] - const badPods = [] - - eachLimit(Object.keys(requestsToMakeGrouped), constants.REQUESTS_IN_PARALLEL, (hashKey, callbackEach) => { - const requestToMake = requestsToMakeGrouped[hashKey] - const toPod = requestToMake.toPod - - this.makeRequest(toPod, requestToMake.endpoint, requestToMake.datas, (err) => { - if (err) { - badPods.push(requestToMake.toPod.id) - return callbackEach() - } - - logger.debug('Removing requests for pod %s.', requestToMake.toPod.id, { requestsIds: requestToMake.ids }) - goodPods.push(requestToMake.toPod.id) - - // Remove the pod id of these request ids - this.getRequestToPodModel().removeByRequestIdsAndPod(requestToMake.ids, requestToMake.toPod.id, callbackEach) - - this.afterRequestHook() - }) - }, () => { - // All the requests were made, we update the pods score - db.Pod.updatePodsScore(goodPods, badPods) - - this.afterRequestsHook() - }) - }) - } - - flush (callback) { - this.getRequestModel().removeAll(callback) - } - - afterRequestHook () { - // Nothing to do, let children reimplement it - } - - afterRequestsHook () { - // Nothing to do, let children reimplement it - } -} diff --git a/server/lib/request/base-request-scheduler.ts b/server/lib/request/base-request-scheduler.ts new file mode 100644 index 000000000..7fc88b5f1 --- /dev/null +++ b/server/lib/request/base-request-scheduler.ts @@ -0,0 +1,154 @@ +import { eachLimit } from 'async/eachLimit' + +const db = require('../../initializers/database') +import { logger, makeSecureRequest } from '../../helpers' +import { + API_VERSION, + REQUESTS_IN_PARALLEL, + REQUESTS_INTERVAL +} from '../../initializers' + +abstract class BaseRequestScheduler { + protected lastRequestTimestamp: number + protected timer: NodeJS.Timer + protected requestInterval: number + protected limitPods: number + protected limitPerPod: number + protected description: string + + constructor () { + this.lastRequestTimestamp = 0 + this.timer = null + this.requestInterval = REQUESTS_INTERVAL + } + + abstract getRequestModel () + abstract getRequestToPodModel () + abstract buildRequestObjects (requests: any) + + activate () { + logger.info('Requests scheduler activated.') + this.lastRequestTimestamp = Date.now() + + this.timer = setInterval(() => { + this.lastRequestTimestamp = Date.now() + this.makeRequests() + }, this.requestInterval) + } + + deactivate () { + logger.info('Requests scheduler deactivated.') + clearInterval(this.timer) + this.timer = null + } + + forceSend () { + logger.info('Force requests scheduler sending.') + this.makeRequests() + } + + remainingMilliSeconds () { + if (this.timer === null) return -1 + + return REQUESTS_INTERVAL - (Date.now() - this.lastRequestTimestamp) + } + + remainingRequestsCount (callback) { + return this.getRequestModel().countTotalRequests(callback) + } + + flush (callback) { + this.getRequestModel().removeAll(callback) + } + + // --------------------------------------------------------------------------- + + // Make a requests to friends of a certain type + protected makeRequest (toPod, requestEndpoint, requestsToMake, callback) { + if (!callback) callback = function () { /* empty */ } + + const params = { + toPod: toPod, + sign: true, // Prove our identity + method: 'POST', + path: '/api/' + API_VERSION + '/remote/' + requestEndpoint, + data: requestsToMake // Requests we need to make + } + + // Make multiple retry requests to all of pods + // The function fire some useful callbacks + makeSecureRequest(params, (err, res) => { + if (err || (res.statusCode !== 200 && res.statusCode !== 201 && res.statusCode !== 204)) { + err = err ? err.message : 'Status code not 20x : ' + res.statusCode + logger.error('Error sending secure request to %s pod.', toPod.host, { error: err }) + + return callback(err) + } + + return callback(null) + }) + } + + // Make all the requests of the scheduler + protected makeRequests () { + this.getRequestModel().listWithLimitAndRandom(this.limitPods, this.limitPerPod, (err, requests) => { + if (err) { + logger.error('Cannot get the list of "%s".', this.description, { err: err }) + return // Abort + } + + // If there are no requests, abort + if (requests.length === 0) { + logger.info('No "%s" to make.', this.description) + return + } + + // We want to group requests by destinations pod and endpoint + const requestsToMakeGrouped = this.buildRequestObjects(requests) + + logger.info('Making "%s" to friends.', this.description) + + const goodPods = [] + const badPods = [] + + eachLimit(Object.keys(requestsToMakeGrouped), REQUESTS_IN_PARALLEL, (hashKey, callbackEach) => { + const requestToMake = requestsToMakeGrouped[hashKey] + const toPod = requestToMake.toPod + + this.makeRequest(toPod, requestToMake.endpoint, requestToMake.datas, (err) => { + if (err) { + badPods.push(requestToMake.toPod.id) + return callbackEach() + } + + logger.debug('Removing requests for pod %s.', requestToMake.toPod.id, { requestsIds: requestToMake.ids }) + goodPods.push(requestToMake.toPod.id) + + // Remove the pod id of these request ids + this.getRequestToPodModel().removeByRequestIdsAndPod(requestToMake.ids, requestToMake.toPod.id, callbackEach) + + this.afterRequestHook() + }) + }, () => { + // All the requests were made, we update the pods score + db.Pod.updatePodsScore(goodPods, badPods) + + this.afterRequestsHook() + }) + }) + } + + protected afterRequestHook () { + // Nothing to do, let children reimplement it + } + + protected afterRequestsHook () { + // Nothing to do, let children reimplement it + } +} + +// --------------------------------------------------------------------------- + +export { + BaseRequestScheduler +} diff --git a/server/lib/request/index.ts b/server/lib/request/index.ts new file mode 100644 index 000000000..c98f956db --- /dev/null +++ b/server/lib/request/index.ts @@ -0,0 +1,3 @@ +export * from './request-scheduler' +export * from './request-video-event-scheduler' +export * from './request-video-qadu-scheduler' diff --git a/server/lib/request/request-scheduler.js b/server/lib/request/request-scheduler.js deleted file mode 100644 index 555ec3e54..000000000 --- a/server/lib/request/request-scheduler.js +++ /dev/null @@ -1,97 +0,0 @@ -'use strict' - -const constants = require('../../initializers/constants') -const BaseRequestScheduler = require('./base-request-scheduler') -const db = require('../../initializers/database') -const logger = require('../../helpers/logger') - -module.exports = class RequestScheduler extends BaseRequestScheduler { - constructor () { - super() - - // We limit the size of the requests - this.limitPods = constants.REQUESTS_LIMIT_PODS - this.limitPerPod = constants.REQUESTS_LIMIT_PER_POD - - this.description = 'requests' - } - - getRequestModel () { - return db.Request - } - - getRequestToPodModel () { - return db.RequestToPod - } - - buildRequestObjects (requests) { - const requestsToMakeGrouped = {} - - Object.keys(requests).forEach(toPodId => { - requests[toPodId].forEach(data => { - const request = data.request - const pod = data.pod - const hashKey = toPodId + request.endpoint - - if (!requestsToMakeGrouped[hashKey]) { - requestsToMakeGrouped[hashKey] = { - toPod: pod, - endpoint: request.endpoint, - ids: [], // request ids, to delete them from the DB in the future - datas: [] // requests data, - } - } - - requestsToMakeGrouped[hashKey].ids.push(request.id) - requestsToMakeGrouped[hashKey].datas.push(request.request) - }) - }) - - return requestsToMakeGrouped - } - - // { type, endpoint, data, toIds, transaction } - createRequest (options, callback) { - const type = options.type - const endpoint = options.endpoint - const data = options.data - const toIds = options.toIds - const transaction = options.transaction - - const pods = [] - - // If there are no destination pods abort - if (toIds.length === 0) return callback(null) - - toIds.forEach(toPod => { - pods.push(db.Pod.build({ id: toPod })) - }) - - const createQuery = { - endpoint, - request: { - type: type, - data: data - } - } - - const dbRequestOptions = { - transaction - } - - return db.Request.create(createQuery, dbRequestOptions).asCallback((err, request) => { - if (err) return callback(err) - - return request.setPods(pods, dbRequestOptions).asCallback(callback) - }) - } - - // --------------------------------------------------------------------------- - - afterRequestsHook () { - // Flush requests with no pod - this.getRequestModel().removeWithEmptyTo(err => { - if (err) logger.error('Error when removing requests with no pods.', { error: err }) - }) - } -} diff --git a/server/lib/request/request-scheduler.ts b/server/lib/request/request-scheduler.ts new file mode 100644 index 000000000..2006a6f03 --- /dev/null +++ b/server/lib/request/request-scheduler.ts @@ -0,0 +1,104 @@ +const db = require('../../initializers/database') +import { BaseRequestScheduler } from './base-request-scheduler' +import { logger } from '../../helpers' +import { + REQUESTS_LIMIT_PODS, + REQUESTS_LIMIT_PER_POD +} from '../../initializers' + +class RequestScheduler extends BaseRequestScheduler { + constructor () { + super() + + // We limit the size of the requests + this.limitPods = REQUESTS_LIMIT_PODS + this.limitPerPod = REQUESTS_LIMIT_PER_POD + + this.description = 'requests' + } + + getRequestModel () { + return db.Request + } + + getRequestToPodModel () { + return db.RequestToPod + } + + buildRequestObjects (requests) { + const requestsToMakeGrouped = {} + + Object.keys(requests).forEach(toPodId => { + requests[toPodId].forEach(data => { + const request = data.request + const pod = data.pod + const hashKey = toPodId + request.endpoint + + if (!requestsToMakeGrouped[hashKey]) { + requestsToMakeGrouped[hashKey] = { + toPod: pod, + endpoint: request.endpoint, + ids: [], // request ids, to delete them from the DB in the future + datas: [] // requests data, + } + } + + requestsToMakeGrouped[hashKey].ids.push(request.id) + requestsToMakeGrouped[hashKey].datas.push(request.request) + }) + }) + + return requestsToMakeGrouped + } + + // { type, endpoint, data, toIds, transaction } + createRequest (options, callback) { + const type = options.type + const endpoint = options.endpoint + const data = options.data + const toIds = options.toIds + const transaction = options.transaction + + const pods = [] + + // If there are no destination pods abort + if (toIds.length === 0) return callback(null) + + toIds.forEach(toPod => { + pods.push(db.Pod.build({ id: toPod })) + }) + + const createQuery = { + endpoint, + request: { + type: type, + data: data + } + } + + const dbRequestOptions = { + transaction + } + + return db.Request.create(createQuery, dbRequestOptions).asCallback((err, request) => { + if (err) return callback(err) + + return request.setPods(pods, dbRequestOptions).asCallback(callback) + }) + } + + // --------------------------------------------------------------------------- + + afterRequestsHook () { + // Flush requests with no pod + this.getRequestModel().removeWithEmptyTo(err => { + if (err) logger.error('Error when removing requests with no pods.', { error: err }) + }) + } +} + +// --------------------------------------------------------------------------- + +export { + RequestScheduler +} diff --git a/server/lib/request/request-video-event-scheduler.js b/server/lib/request/request-video-event-scheduler.js deleted file mode 100644 index e54d34f4a..000000000 --- a/server/lib/request/request-video-event-scheduler.js +++ /dev/null @@ -1,108 +0,0 @@ -'use strict' - -const BaseRequestScheduler = require('./base-request-scheduler') -const constants = require('../../initializers/constants') -const db = require('../../initializers/database') - -module.exports = class RequestVideoEventScheduler extends BaseRequestScheduler { - constructor () { - super() - - // We limit the size of the requests - this.limitPods = constants.REQUESTS_VIDEO_EVENT_LIMIT_PODS - this.limitPerPod = constants.REQUESTS_VIDEO_EVENT_LIMIT_PER_POD - - this.description = 'video event requests' - } - - getRequestModel () { - return db.RequestVideoEvent - } - - getRequestToPodModel () { - return db.RequestVideoEvent - } - - buildRequestObjects (eventsToProcess) { - const requestsToMakeGrouped = {} - - /* Example: - { - pod1: { - video1: { views: 4, likes: 5 }, - video2: { likes: 5 } - } - } - */ - const eventsPerVideoPerPod = {} - - // We group video events per video and per pod - // We add the counts of the same event types - Object.keys(eventsToProcess).forEach(toPodId => { - eventsToProcess[toPodId].forEach(eventToProcess => { - if (!eventsPerVideoPerPod[toPodId]) eventsPerVideoPerPod[toPodId] = {} - - if (!requestsToMakeGrouped[toPodId]) { - requestsToMakeGrouped[toPodId] = { - toPod: eventToProcess.pod, - endpoint: constants.REQUEST_VIDEO_EVENT_ENDPOINT, - ids: [], // request ids, to delete them from the DB in the future - datas: [] // requests data - } - } - requestsToMakeGrouped[toPodId].ids.push(eventToProcess.id) - - const eventsPerVideo = eventsPerVideoPerPod[toPodId] - const remoteId = eventToProcess.video.remoteId - if (!eventsPerVideo[remoteId]) eventsPerVideo[remoteId] = {} - - const events = eventsPerVideo[remoteId] - if (!events[eventToProcess.type]) events[eventToProcess.type] = 0 - - events[eventToProcess.type] += eventToProcess.count - }) - }) - - // Now we build our requests array per pod - Object.keys(eventsPerVideoPerPod).forEach(toPodId => { - const eventsForPod = eventsPerVideoPerPod[toPodId] - - Object.keys(eventsForPod).forEach(remoteId => { - const eventsForVideo = eventsForPod[remoteId] - - Object.keys(eventsForVideo).forEach(eventType => { - requestsToMakeGrouped[toPodId].datas.push({ - data: { - remoteId, - eventType, - count: eventsForVideo[eventType] - } - }) - }) - }) - }) - - return requestsToMakeGrouped - } - - // { type, videoId, count?, transaction? } - createRequest (options, callback) { - const type = options.type - const videoId = options.videoId - const transaction = options.transaction - let count = options.count - - if (count === undefined) count = 1 - - const dbRequestOptions = {} - if (transaction) dbRequestOptions.transaction = transaction - - const createQuery = { - type, - count, - videoId - } - - return db.RequestVideoEvent.create(createQuery, dbRequestOptions).asCallback(callback) - } -} diff --git a/server/lib/request/request-video-event-scheduler.ts b/server/lib/request/request-video-event-scheduler.ts new file mode 100644 index 000000000..6e5306c7d --- /dev/null +++ b/server/lib/request/request-video-event-scheduler.ts @@ -0,0 +1,116 @@ +const db = require('../../initializers/database') +import { BaseRequestScheduler } from './base-request-scheduler' +import { + REQUESTS_VIDEO_EVENT_LIMIT_PODS, + REQUESTS_VIDEO_EVENT_LIMIT_PER_POD, + REQUEST_VIDEO_EVENT_ENDPOINT +} from '../../initializers' + +class RequestVideoEventScheduler extends BaseRequestScheduler { + constructor () { + super() + + // We limit the size of the requests + this.limitPods = REQUESTS_VIDEO_EVENT_LIMIT_PODS + this.limitPerPod = REQUESTS_VIDEO_EVENT_LIMIT_PER_POD + + this.description = 'video event requests' + } + + getRequestModel () { + return db.RequestVideoEvent + } + + getRequestToPodModel () { + return db.RequestVideoEvent + } + + buildRequestObjects (eventsToProcess) { + const requestsToMakeGrouped = {} + + /* Example: + { + pod1: { + video1: { views: 4, likes: 5 }, + video2: { likes: 5 } + } + } + */ + const eventsPerVideoPerPod = {} + + // We group video events per video and per pod + // We add the counts of the same event types + Object.keys(eventsToProcess).forEach(toPodId => { + eventsToProcess[toPodId].forEach(eventToProcess => { + if (!eventsPerVideoPerPod[toPodId]) eventsPerVideoPerPod[toPodId] = {} + + if (!requestsToMakeGrouped[toPodId]) { + requestsToMakeGrouped[toPodId] = { + toPod: eventToProcess.pod, + endpoint: REQUEST_VIDEO_EVENT_ENDPOINT, + ids: [], // request ids, to delete them from the DB in the future + datas: [] // requests data + } + } + requestsToMakeGrouped[toPodId].ids.push(eventToProcess.id) + + const eventsPerVideo = eventsPerVideoPerPod[toPodId] + const remoteId = eventToProcess.video.remoteId + if (!eventsPerVideo[remoteId]) eventsPerVideo[remoteId] = {} + + const events = eventsPerVideo[remoteId] + if (!events[eventToProcess.type]) events[eventToProcess.type] = 0 + + events[eventToProcess.type] += eventToProcess.count + }) + }) + + // Now we build our requests array per pod + Object.keys(eventsPerVideoPerPod).forEach(toPodId => { + const eventsForPod = eventsPerVideoPerPod[toPodId] + + Object.keys(eventsForPod).forEach(remoteId => { + const eventsForVideo = eventsForPod[remoteId] + + Object.keys(eventsForVideo).forEach(eventType => { + requestsToMakeGrouped[toPodId].datas.push({ + data: { + remoteId, + eventType, + count: eventsForVideo[eventType] + } + }) + }) + }) + }) + + return requestsToMakeGrouped + } + + // { type, videoId, count?, transaction? } + createRequest (options, callback) { + const type = options.type + const videoId = options.videoId + const transaction = options.transaction + let count = options.count + + if (count === undefined) count = 1 + + const dbRequestOptions: { transaction?: any } = {} + if (transaction) dbRequestOptions.transaction = transaction + + const createQuery = { + type, + count, + videoId + } + + return db.RequestVideoEvent.create(createQuery, dbRequestOptions).asCallback(callback) + } +} + +// --------------------------------------------------------------------------- + +export { + RequestVideoEventScheduler +} diff --git a/server/lib/request/request-video-qadu-scheduler.js b/server/lib/request/request-video-qadu-scheduler.js deleted file mode 100644 index 17402b556..000000000 --- a/server/lib/request/request-video-qadu-scheduler.js +++ /dev/null @@ -1,117 +0,0 @@ -'use strict' - -const BaseRequestScheduler = require('./base-request-scheduler') -const constants = require('../../initializers/constants') -const db = require('../../initializers/database') -const logger = require('../../helpers/logger') - -module.exports = class RequestVideoQaduScheduler extends BaseRequestScheduler { - constructor () { - super() - - // We limit the size of the requests - this.limitPods = constants.REQUESTS_VIDEO_QADU_LIMIT_PODS - this.limitPerPod = constants.REQUESTS_VIDEO_QADU_LIMIT_PER_POD - - this.description = 'video QADU requests' - } - - getRequestModel () { - return db.RequestVideoQadu - } - - getRequestToPodModel () { - return db.RequestVideoQadu - } - - buildRequestObjects (requests) { - const requestsToMakeGrouped = {} - - Object.keys(requests).forEach(toPodId => { - requests[toPodId].forEach(data => { - const request = data.request - const video = data.video - const pod = data.pod - const hashKey = toPodId - - if (!requestsToMakeGrouped[hashKey]) { - requestsToMakeGrouped[hashKey] = { - toPod: pod, - endpoint: constants.REQUEST_VIDEO_QADU_ENDPOINT, - ids: [], // request ids, to delete them from the DB in the future - datas: [], // requests data - videos: {} - } - } - - // Maybe another attribute was filled for this video - let videoData = requestsToMakeGrouped[hashKey].videos[video.id] - if (!videoData) videoData = {} - - switch (request.type) { - case constants.REQUEST_VIDEO_QADU_TYPES.LIKES: - videoData.likes = video.likes - break - - case constants.REQUEST_VIDEO_QADU_TYPES.DISLIKES: - videoData.dislikes = video.dislikes - break - - case constants.REQUEST_VIDEO_QADU_TYPES.VIEWS: - videoData.views = video.views - break - - default: - logger.error('Unknown request video QADU type %s.', request.type) - return - } - - // Do not forget the remoteId so the remote pod can identify the video - videoData.remoteId = video.id - requestsToMakeGrouped[hashKey].ids.push(request.id) - - // Maybe there are multiple quick and dirty update for the same video - // We use this hashmap to dedupe them - requestsToMakeGrouped[hashKey].videos[video.id] = videoData - }) - }) - - // Now we deduped similar quick and dirty updates, we can build our requests datas - Object.keys(requestsToMakeGrouped).forEach(hashKey => { - Object.keys(requestsToMakeGrouped[hashKey].videos).forEach(videoId => { - const videoData = requestsToMakeGrouped[hashKey].videos[videoId] - - requestsToMakeGrouped[hashKey].datas.push({ - data: videoData - }) - }) - - // We don't need it anymore, it was just to build our datas array - delete requestsToMakeGrouped[hashKey].videos - }) - - return requestsToMakeGrouped - } - - // { type, videoId, transaction? } - createRequest (options, callback) { - const type = options.type - const videoId = options.videoId - const transaction = options.transaction - - const dbRequestOptions = {} - if (transaction) dbRequestOptions.transaction = transaction - - // Send the update to all our friends - db.Pod.listAllIds(options.transaction, function (err, podIds) { - if (err) return callback(err) - - const queries = [] - podIds.forEach(podId => { - queries.push({ type, videoId, podId }) - }) - - return db.RequestVideoQadu.bulkCreate(queries, dbRequestOptions).asCallback(callback) - }) - } -} diff --git a/server/lib/request/request-video-qadu-scheduler.ts b/server/lib/request/request-video-qadu-scheduler.ts new file mode 100644 index 000000000..d81822723 --- /dev/null +++ b/server/lib/request/request-video-qadu-scheduler.ts @@ -0,0 +1,126 @@ +const db = require('../../initializers/database') +import { BaseRequestScheduler } from './base-request-scheduler' +import { logger } from '../../helpers' +import { + REQUESTS_VIDEO_QADU_LIMIT_PODS, + REQUESTS_VIDEO_QADU_LIMIT_PER_POD, + REQUEST_VIDEO_QADU_ENDPOINT, + REQUEST_VIDEO_QADU_TYPES +} from '../../initializers' + +class RequestVideoQaduScheduler extends BaseRequestScheduler { + constructor () { + super() + + // We limit the size of the requests + this.limitPods = REQUESTS_VIDEO_QADU_LIMIT_PODS + this.limitPerPod = REQUESTS_VIDEO_QADU_LIMIT_PER_POD + + this.description = 'video QADU requests' + } + + getRequestModel () { + return db.RequestVideoQadu + } + + getRequestToPodModel () { + return db.RequestVideoQadu + } + + buildRequestObjects (requests) { + const requestsToMakeGrouped = {} + + Object.keys(requests).forEach(toPodId => { + requests[toPodId].forEach(data => { + const request = data.request + const video = data.video + const pod = data.pod + const hashKey = toPodId + + if (!requestsToMakeGrouped[hashKey]) { + requestsToMakeGrouped[hashKey] = { + toPod: pod, + endpoint: REQUEST_VIDEO_QADU_ENDPOINT, + ids: [], // request ids, to delete them from the DB in the future + datas: [], // requests data + videos: {} + } + } + + // Maybe another attribute was filled for this video + let videoData = requestsToMakeGrouped[hashKey].videos[video.id] + if (!videoData) videoData = {} + + switch (request.type) { + case REQUEST_VIDEO_QADU_TYPES.LIKES: + videoData.likes = video.likes + break + + case REQUEST_VIDEO_QADU_TYPES.DISLIKES: + videoData.dislikes = video.dislikes + break + + case REQUEST_VIDEO_QADU_TYPES.VIEWS: + videoData.views = video.views + break + + default: + logger.error('Unknown request video QADU type %s.', request.type) + return + } + + // Do not forget the remoteId so the remote pod can identify the video + videoData.remoteId = video.id + requestsToMakeGrouped[hashKey].ids.push(request.id) + + // Maybe there are multiple quick and dirty update for the same video + // We use this hashmap to dedupe them + requestsToMakeGrouped[hashKey].videos[video.id] = videoData + }) + }) + + // Now we deduped similar quick and dirty updates, we can build our requests datas + Object.keys(requestsToMakeGrouped).forEach(hashKey => { + Object.keys(requestsToMakeGrouped[hashKey].videos).forEach(videoId => { + const videoData = requestsToMakeGrouped[hashKey].videos[videoId] + + requestsToMakeGrouped[hashKey].datas.push({ + data: videoData + }) + }) + + // We don't need it anymore, it was just to build our datas array + delete requestsToMakeGrouped[hashKey].videos + }) + + return requestsToMakeGrouped + } + + // { type, videoId, transaction? } + createRequest (options, callback) { + const type = options.type + const videoId = options.videoId + const transaction = options.transaction + + const dbRequestOptions: { transaction?: any } = {} + if (transaction) dbRequestOptions.transaction = transaction + + // Send the update to all our friends + db.Pod.listAllIds(options.transaction, function (err, podIds) { + if (err) return callback(err) + + const queries = [] + podIds.forEach(podId => { + queries.push({ type, videoId, podId }) + }) + + return db.RequestVideoQadu.bulkCreate(queries, dbRequestOptions).asCallback(callback) + }) + } +} + +// --------------------------------------------------------------------------- + +export { + RequestVideoQaduScheduler +} diff --git a/server/middlewares/admin.js b/server/middlewares/admin.js deleted file mode 100644 index 3288f4c6b..000000000 --- a/server/middlewares/admin.js +++ /dev/null @@ -1,21 +0,0 @@ -'use strict' - -const logger = require('../helpers/logger') - -const adminMiddleware = { - ensureIsAdmin -} - -function ensureIsAdmin (req, res, next) { - const user = res.locals.oauth.token.user - if (user.isAdmin() === false) { - logger.info('A non admin user is trying to access to an admin content.') - return res.sendStatus(403) - } - - return next() -} - -// --------------------------------------------------------------------------- - -module.exports = adminMiddleware diff --git a/server/middlewares/admin.ts b/server/middlewares/admin.ts new file mode 100644 index 000000000..ebafa36a4 --- /dev/null +++ b/server/middlewares/admin.ts @@ -0,0 +1,17 @@ +const logger = require('../helpers/logger') + +function ensureIsAdmin (req, res, next) { + const user = res.locals.oauth.token.user + if (user.isAdmin() === false) { + logger.info('A non admin user is trying to access to an admin content.') + return res.sendStatus(403) + } + + return next() +} + +// --------------------------------------------------------------------------- + +export { + ensureIsAdmin +} diff --git a/server/middlewares/index.js b/server/middlewares/index.js deleted file mode 100644 index 3f253e31b..000000000 --- a/server/middlewares/index.js +++ /dev/null @@ -1,25 +0,0 @@ -'use strict' - -const adminMiddleware = require('./admin') -const oauthMiddleware = require('./oauth') -const paginationMiddleware = require('./pagination') -const podsMiddleware = require('./pods') -const validatorsMiddleware = require('./validators') -const searchMiddleware = require('./search') -const sortMiddleware = require('./sort') -const secureMiddleware = require('./secure') - -const middlewares = { - admin: adminMiddleware, - oauth: oauthMiddleware, - pagination: paginationMiddleware, - pods: podsMiddleware, - search: searchMiddleware, - secure: secureMiddleware, - sort: sortMiddleware, - validators: validatorsMiddleware -} - -// --------------------------------------------------------------------------- - -module.exports = middlewares diff --git a/server/middlewares/index.ts b/server/middlewares/index.ts new file mode 100644 index 000000000..2c1c5fa53 --- /dev/null +++ b/server/middlewares/index.ts @@ -0,0 +1,8 @@ +export * from './validators'; +export * from './admin'; +export * from './oauth'; +export * from './pagination'; +export * from './pods'; +export * from './search'; +export * from './secure'; +export * from './sort'; diff --git a/server/middlewares/oauth.js b/server/middlewares/oauth.js deleted file mode 100644 index 3a02b9b48..000000000 --- a/server/middlewares/oauth.js +++ /dev/null @@ -1,38 +0,0 @@ -'use strict' - -const OAuthServer = require('express-oauth-server') - -const constants = require('../initializers/constants') -const logger = require('../helpers/logger') - -const oAuthServer = new OAuthServer({ - accessTokenLifetime: constants.OAUTH_LIFETIME.ACCESS_TOKEN, - refreshTokenLifetime: constants.OAUTH_LIFETIME.REFRESH_TOKEN, - model: require('../lib/oauth-model') -}) - -const oAuth = { - authenticate, - token -} - -function authenticate (req, res, next) { - oAuthServer.authenticate()(req, res, function (err) { - if (err) { - logger.error('Cannot authenticate.', { error: err }) - return res.sendStatus(500) - } - - if (res.statusCode === 401 || res.statusCode === 400 || res.statusCode === 503) return res.end() - - return next() - }) -} - -function token (req, res, next) { - return oAuthServer.token()(req, res, next) -} - -// --------------------------------------------------------------------------- - -module.exports = oAuth diff --git a/server/middlewares/oauth.ts b/server/middlewares/oauth.ts new file mode 100644 index 000000000..31ae1e000 --- /dev/null +++ b/server/middlewares/oauth.ts @@ -0,0 +1,34 @@ +import OAuthServer = require('express-oauth-server') + +const constants = require('../initializers/constants') +const logger = require('../helpers/logger') + +const oAuthServer = new OAuthServer({ + accessTokenLifetime: constants.OAUTH_LIFETIME.ACCESS_TOKEN, + refreshTokenLifetime: constants.OAUTH_LIFETIME.REFRESH_TOKEN, + model: require('../lib/oauth-model') +}) + +function authenticate (req, res, next) { + oAuthServer.authenticate()(req, res, function (err) { + if (err) { + logger.error('Cannot authenticate.', { error: err }) + return res.sendStatus(500) + } + + if (res.statusCode === 401 || res.statusCode === 400 || res.statusCode === 503) return res.end() + + return next() + }) +} + +function token (req, res, next) { + return oAuthServer.token()(req, res, next) +} + +// --------------------------------------------------------------------------- + +export { + authenticate, + token +} diff --git a/server/middlewares/pagination.js b/server/middlewares/pagination.js deleted file mode 100644 index a90f60aab..000000000 --- a/server/middlewares/pagination.js +++ /dev/null @@ -1,20 +0,0 @@ -'use strict' - -const constants = require('../initializers/constants') - -const paginationMiddleware = { - setPagination -} - -function setPagination (req, res, next) { - if (!req.query.start) req.query.start = 0 - else req.query.start = parseInt(req.query.start, 10) - if (!req.query.count) req.query.count = constants.PAGINATION_COUNT_DEFAULT - else req.query.count = parseInt(req.query.count, 10) - - return next() -} - -// --------------------------------------------------------------------------- - -module.exports = paginationMiddleware diff --git a/server/middlewares/pagination.ts b/server/middlewares/pagination.ts new file mode 100644 index 000000000..8fe9f9082 --- /dev/null +++ b/server/middlewares/pagination.ts @@ -0,0 +1,17 @@ +const constants = require('../initializers/constants') + +function setPagination (req, res, next) { + if (!req.query.start) req.query.start = 0 + else req.query.start = parseInt(req.query.start, 10) + + if (!req.query.count) req.query.count = constants.PAGINATION_COUNT_DEFAULT + else req.query.count = parseInt(req.query.count, 10) + + return next() +} + +// --------------------------------------------------------------------------- + +export { + setPagination +} diff --git a/server/middlewares/pods.js b/server/middlewares/pods.js deleted file mode 100644 index 2647f9ff0..000000000 --- a/server/middlewares/pods.js +++ /dev/null @@ -1,59 +0,0 @@ -'use strict' - -const constants = require('../initializers/constants') - -const podsMiddleware = { - setBodyHostsPort, - setBodyHostPort -} - -function setBodyHostsPort (req, res, next) { - if (!req.body.hosts) return next() - - for (let i = 0; i < req.body.hosts.length; i++) { - const hostWithPort = getHostWithPort(req.body.hosts[i]) - - // Problem with the url parsing? - if (hostWithPort === null) { - return res.sendStatus(500) - } - - req.body.hosts[i] = hostWithPort - } - - return next() -} - -function setBodyHostPort (req, res, next) { - if (!req.body.host) return next() - - const hostWithPort = getHostWithPort(req.body.host) - - // Problem with the url parsing? - if (hostWithPort === null) { - return res.sendStatus(500) - } - - req.body.host = hostWithPort - - return next() -} - -// --------------------------------------------------------------------------- - -module.exports = podsMiddleware - -// --------------------------------------------------------------------------- - -function getHostWithPort (host) { - const splitted = host.split(':') - - // The port was not specified - if (splitted.length === 1) { - if (constants.REMOTE_SCHEME.HTTP === 'https') return host + ':443' - - return host + ':80' - } - - return host -} diff --git a/server/middlewares/pods.ts b/server/middlewares/pods.ts new file mode 100644 index 000000000..e405f265e --- /dev/null +++ b/server/middlewares/pods.ts @@ -0,0 +1,57 @@ +'use strict' + +const constants = require('../initializers/constants') + +function setBodyHostsPort (req, res, next) { + if (!req.body.hosts) return next() + + for (let i = 0; i < req.body.hosts.length; i++) { + const hostWithPort = getHostWithPort(req.body.hosts[i]) + + // Problem with the url parsing? + if (hostWithPort === null) { + return res.sendStatus(500) + } + + req.body.hosts[i] = hostWithPort + } + + return next() +} + +function setBodyHostPort (req, res, next) { + if (!req.body.host) return next() + + const hostWithPort = getHostWithPort(req.body.host) + + // Problem with the url parsing? + if (hostWithPort === null) { + return res.sendStatus(500) + } + + req.body.host = hostWithPort + + return next() +} + +// --------------------------------------------------------------------------- + +export { + setBodyHostsPort, + setBodyHostPort +} + +// --------------------------------------------------------------------------- + +function getHostWithPort (host) { + const splitted = host.split(':') + + // The port was not specified + if (splitted.length === 1) { + if (constants.REMOTE_SCHEME.HTTP === 'https') return host + ':443' + + return host + ':80' + } + + return host +} diff --git a/server/middlewares/search.js b/server/middlewares/search.js deleted file mode 100644 index bb88faf54..000000000 --- a/server/middlewares/search.js +++ /dev/null @@ -1,15 +0,0 @@ -'use strict' - -const searchMiddleware = { - setVideosSearch -} - -function setVideosSearch (req, res, next) { - if (!req.query.field) req.query.field = 'name' - - return next() -} - -// --------------------------------------------------------------------------- - -module.exports = searchMiddleware diff --git a/server/middlewares/search.ts b/server/middlewares/search.ts new file mode 100644 index 000000000..05a2e7442 --- /dev/null +++ b/server/middlewares/search.ts @@ -0,0 +1,11 @@ +function setVideosSearch (req, res, next) { + if (!req.query.field) req.query.field = 'name' + + return next() +} + +// --------------------------------------------------------------------------- + +export { + setVideosSearch +} diff --git a/server/middlewares/secure.js b/server/middlewares/secure.js deleted file mode 100644 index 7c5c72508..000000000 --- a/server/middlewares/secure.js +++ /dev/null @@ -1,52 +0,0 @@ -'use strict' - -const db = require('../initializers/database') -const logger = require('../helpers/logger') -const peertubeCrypto = require('../helpers/peertube-crypto') - -const secureMiddleware = { - checkSignature -} - -function checkSignature (req, res, next) { - const host = req.body.signature.host - db.Pod.loadByHost(host, function (err, pod) { - if (err) { - logger.error('Cannot get signed host in body.', { error: err }) - return res.sendStatus(500) - } - - if (pod === null) { - logger.error('Unknown pod %s.', host) - return res.sendStatus(403) - } - - logger.debug('Checking signature from %s.', host) - - let signatureShouldBe - // If there is data in the body the sender used it for its signature - // If there is no data we just use its host as signature - if (req.body.data) { - signatureShouldBe = req.body.data - } else { - signatureShouldBe = host - } - - const signatureOk = peertubeCrypto.checkSignature(pod.publicKey, signatureShouldBe, req.body.signature.signature) - - if (signatureOk === true) { - res.locals.secure = { - pod - } - - return next() - } - - logger.error('Signature is not okay in body for %s.', req.body.signature.host) - return res.sendStatus(403) - }) -} - -// --------------------------------------------------------------------------- - -module.exports = secureMiddleware diff --git a/server/middlewares/secure.ts b/server/middlewares/secure.ts new file mode 100644 index 000000000..ee8545028 --- /dev/null +++ b/server/middlewares/secure.ts @@ -0,0 +1,48 @@ +const db = require('../initializers/database') +const logger = require('../helpers/logger') +const peertubeCrypto = require('../helpers/peertube-crypto') + +function checkSignature (req, res, next) { + const host = req.body.signature.host + db.Pod.loadByHost(host, function (err, pod) { + if (err) { + logger.error('Cannot get signed host in body.', { error: err }) + return res.sendStatus(500) + } + + if (pod === null) { + logger.error('Unknown pod %s.', host) + return res.sendStatus(403) + } + + logger.debug('Checking signature from %s.', host) + + let signatureShouldBe + // If there is data in the body the sender used it for its signature + // If there is no data we just use its host as signature + if (req.body.data) { + signatureShouldBe = req.body.data + } else { + signatureShouldBe = host + } + + const signatureOk = peertubeCrypto.checkSignature(pod.publicKey, signatureShouldBe, req.body.signature.signature) + + if (signatureOk === true) { + res.locals.secure = { + pod + } + + return next() + } + + logger.error('Signature is not okay in body for %s.', req.body.signature.host) + return res.sendStatus(403) + }) +} + +// --------------------------------------------------------------------------- + +export { + checkSignature +} diff --git a/server/middlewares/sort.js b/server/middlewares/sort.js deleted file mode 100644 index 39e167265..000000000 --- a/server/middlewares/sort.js +++ /dev/null @@ -1,29 +0,0 @@ -'use strict' - -const sortMiddleware = { - setUsersSort, - setVideoAbusesSort, - setVideosSort -} - -function setUsersSort (req, res, next) { - if (!req.query.sort) req.query.sort = '-createdAt' - - return next() -} - -function setVideoAbusesSort (req, res, next) { - if (!req.query.sort) req.query.sort = '-createdAt' - - return next() -} - -function setVideosSort (req, res, next) { - if (!req.query.sort) req.query.sort = '-createdAt' - - return next() -} - -// --------------------------------------------------------------------------- - -module.exports = sortMiddleware diff --git a/server/middlewares/sort.ts b/server/middlewares/sort.ts new file mode 100644 index 000000000..ab9ccf524 --- /dev/null +++ b/server/middlewares/sort.ts @@ -0,0 +1,25 @@ +function setUsersSort (req, res, next) { + if (!req.query.sort) req.query.sort = '-createdAt' + + return next() +} + +function setVideoAbusesSort (req, res, next) { + if (!req.query.sort) req.query.sort = '-createdAt' + + return next() +} + +function setVideosSort (req, res, next) { + if (!req.query.sort) req.query.sort = '-createdAt' + + return next() +} + +// --------------------------------------------------------------------------- + +export { + setUsersSort, + setVideoAbusesSort, + setVideosSort +} diff --git a/server/middlewares/validators/index.js b/server/middlewares/validators/index.js deleted file mode 100644 index 6c3a9c2b4..000000000 --- a/server/middlewares/validators/index.js +++ /dev/null @@ -1,21 +0,0 @@ -'use strict' - -const paginationValidators = require('./pagination') -const podsValidators = require('./pods') -const remoteValidators = require('./remote') -const sortValidators = require('./sort') -const usersValidators = require('./users') -const videosValidators = require('./videos') - -const validators = { - pagination: paginationValidators, - pods: podsValidators, - remote: remoteValidators, - sort: sortValidators, - users: usersValidators, - videos: videosValidators -} - -// --------------------------------------------------------------------------- - -module.exports = validators diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts new file mode 100644 index 000000000..42ba465ec --- /dev/null +++ b/server/middlewares/validators/index.ts @@ -0,0 +1,6 @@ +export * from './remote' +export * from './pagination' +export * from './pods' +export * from './sort' +export * from './users' +export * from './videos' diff --git a/server/middlewares/validators/pagination.js b/server/middlewares/validators/pagination.js deleted file mode 100644 index 16682696e..000000000 --- a/server/middlewares/validators/pagination.js +++ /dev/null @@ -1,21 +0,0 @@ -'use strict' - -const checkErrors = require('./utils').checkErrors -const logger = require('../../helpers/logger') - -const validatorsPagination = { - pagination -} - -function pagination (req, res, next) { - req.checkQuery('start', 'Should have a number start').optional().isInt() - req.checkQuery('count', 'Should have a number count').optional().isInt() - - logger.debug('Checking pagination parameters', { parameters: req.query }) - - checkErrors(req, res, next) -} - -// --------------------------------------------------------------------------- - -module.exports = validatorsPagination diff --git a/server/middlewares/validators/pagination.ts b/server/middlewares/validators/pagination.ts new file mode 100644 index 000000000..de719c05b --- /dev/null +++ b/server/middlewares/validators/pagination.ts @@ -0,0 +1,17 @@ +import { checkErrors } from './utils' +import { logger } from '../../helpers' + +function paginationValidator (req, res, next) { + req.checkQuery('start', 'Should have a number start').optional().isInt() + req.checkQuery('count', 'Should have a number count').optional().isInt() + + logger.debug('Checking pagination parameters', { parameters: req.query }) + + checkErrors(req, res, next) +} + +// --------------------------------------------------------------------------- + +export { + paginationValidator +} diff --git a/server/middlewares/validators/pods.js b/server/middlewares/validators/pods.js deleted file mode 100644 index 0bf4b1844..000000000 --- a/server/middlewares/validators/pods.js +++ /dev/null @@ -1,67 +0,0 @@ -'use strict' - -const checkErrors = require('./utils').checkErrors -const constants = require('../../initializers/constants') -const db = require('../../initializers/database') -const friends = require('../../lib/friends') -const logger = require('../../helpers/logger') -const utils = require('../../helpers/utils') - -const validatorsPod = { - makeFriends, - podsAdd -} - -function makeFriends (req, res, next) { - // Force https if the administrator wants to make friends - if (utils.isTestInstance() === false && constants.CONFIG.WEBSERVER.SCHEME === 'http') { - return res.status(400).send('Cannot make friends with a non HTTPS webserver.') - } - - req.checkBody('hosts', 'Should have an array of unique hosts').isEachUniqueHostValid() - - logger.debug('Checking makeFriends parameters', { parameters: req.body }) - - checkErrors(req, res, function () { - friends.hasFriends(function (err, hasFriends) { - if (err) { - logger.error('Cannot know if we have friends.', { error: err }) - res.sendStatus(500) - } - - if (hasFriends === true) { - // We need to quit our friends before make new ones - return res.sendStatus(409) - } - - return next() - }) - }) -} - -function podsAdd (req, res, next) { - req.checkBody('host', 'Should have a host').isHostValid() - req.checkBody('email', 'Should have an email').isEmail() - req.checkBody('publicKey', 'Should have a public key').notEmpty() - logger.debug('Checking podsAdd parameters', { parameters: req.body }) - - checkErrors(req, res, function () { - db.Pod.loadByHost(req.body.host, function (err, pod) { - if (err) { - logger.error('Cannot load pod by host.', { error: err }) - res.sendStatus(500) - } - - // Pod with this host already exists - if (pod) { - return res.sendStatus(409) - } - - return next() - }) - }) -} - -// --------------------------------------------------------------------------- - -module.exports = validatorsPod diff --git a/server/middlewares/validators/pods.ts b/server/middlewares/validators/pods.ts new file mode 100644 index 000000000..fbfd268d0 --- /dev/null +++ b/server/middlewares/validators/pods.ts @@ -0,0 +1,63 @@ +const db = require('../../initializers/database') +import { checkErrors } from './utils' +import { logger } from '../../helpers' +import { CONFIG } from '../../initializers' +import { hasFriends } from '../../lib' +import { isTestInstance } from '../../helpers' + +function makeFriendsValidator (req, res, next) { + // Force https if the administrator wants to make friends + if (isTestInstance() === false && CONFIG.WEBSERVER.SCHEME === 'http') { + return res.status(400).send('Cannot make friends with a non HTTPS webserver.') + } + + req.checkBody('hosts', 'Should have an array of unique hosts').isEachUniqueHostValid() + + logger.debug('Checking makeFriends parameters', { parameters: req.body }) + + checkErrors(req, res, function () { + hasFriends(function (err, heHasFriends) { + if (err) { + logger.error('Cannot know if we have friends.', { error: err }) + res.sendStatus(500) + } + + if (heHasFriends === true) { + // We need to quit our friends before make new ones + return res.sendStatus(409) + } + + return next() + }) + }) +} + +function podsAddValidator (req, res, next) { + req.checkBody('host', 'Should have a host').isHostValid() + req.checkBody('email', 'Should have an email').isEmail() + req.checkBody('publicKey', 'Should have a public key').notEmpty() + logger.debug('Checking podsAdd parameters', { parameters: req.body }) + + checkErrors(req, res, function () { + db.Pod.loadByHost(req.body.host, function (err, pod) { + if (err) { + logger.error('Cannot load pod by host.', { error: err }) + res.sendStatus(500) + } + + // Pod with this host already exists + if (pod) { + return res.sendStatus(409) + } + + return next() + }) + }) +} + +// --------------------------------------------------------------------------- + +export { + makeFriendsValidator, + podsAddValidator +} diff --git a/server/middlewares/validators/remote/index.js b/server/middlewares/validators/remote/index.js deleted file mode 100644 index 022a2fe50..000000000 --- a/server/middlewares/validators/remote/index.js +++ /dev/null @@ -1,13 +0,0 @@ -'use strict' - -const remoteSignatureValidators = require('./signature') -const remoteVideosValidators = require('./videos') - -const validators = { - signature: remoteSignatureValidators, - videos: remoteVideosValidators -} - -// --------------------------------------------------------------------------- - -module.exports = validators diff --git a/server/middlewares/validators/remote/index.ts b/server/middlewares/validators/remote/index.ts new file mode 100644 index 000000000..d0d7740b1 --- /dev/null +++ b/server/middlewares/validators/remote/index.ts @@ -0,0 +1,2 @@ +export * from './signature' +export * from './videos' diff --git a/server/middlewares/validators/remote/signature.js b/server/middlewares/validators/remote/signature.js deleted file mode 100644 index 002232c05..000000000 --- a/server/middlewares/validators/remote/signature.js +++ /dev/null @@ -1,21 +0,0 @@ -'use strict' - -const checkErrors = require('../utils').checkErrors -const logger = require('../../../helpers/logger') - -const validatorsRemoteSignature = { - signature -} - -function signature (req, res, next) { - req.checkBody('signature.host', 'Should have a signature host').isURL() - req.checkBody('signature.signature', 'Should have a signature').notEmpty() - - logger.debug('Checking signature parameters', { parameters: { signature: req.body.signature } }) - - checkErrors(req, res, next) -} - -// --------------------------------------------------------------------------- - -module.exports = validatorsRemoteSignature diff --git a/server/middlewares/validators/remote/signature.ts b/server/middlewares/validators/remote/signature.ts new file mode 100644 index 000000000..6e3ebe7db --- /dev/null +++ b/server/middlewares/validators/remote/signature.ts @@ -0,0 +1,17 @@ +import { logger } from '../../../helpers' +import { checkErrors } from '../utils' + +function signatureValidator (req, res, next) { + req.checkBody('signature.host', 'Should have a signature host').isURL() + req.checkBody('signature.signature', 'Should have a signature').notEmpty() + + logger.debug('Checking signature parameters', { parameters: { signature: req.body.signature } }) + + checkErrors(req, res, next) +} + +// --------------------------------------------------------------------------- + +export { + signatureValidator +} diff --git a/server/middlewares/validators/remote/videos.js b/server/middlewares/validators/remote/videos.js deleted file mode 100644 index f2c6cba5e..000000000 --- a/server/middlewares/validators/remote/videos.js +++ /dev/null @@ -1,37 +0,0 @@ -'use strict' - -const checkErrors = require('../utils').checkErrors -const logger = require('../../../helpers/logger') - -const validatorsRemoteVideos = { - remoteVideos, - remoteQaduVideos, - remoteEventsVideos -} - -function remoteVideos (req, res, next) { - req.checkBody('data').isEachRemoteRequestVideosValid() - - logger.debug('Checking remoteVideos parameters', { parameters: req.body }) - - checkErrors(req, res, next) -} - -function remoteQaduVideos (req, res, next) { - req.checkBody('data').isEachRemoteRequestVideosQaduValid() - - logger.debug('Checking remoteQaduVideos parameters', { parameters: req.body }) - - checkErrors(req, res, next) -} - -function remoteEventsVideos (req, res, next) { - req.checkBody('data').isEachRemoteRequestVideosEventsValid() - - logger.debug('Checking remoteEventsVideos parameters', { parameters: req.body }) - - checkErrors(req, res, next) -} -// --------------------------------------------------------------------------- - -module.exports = validatorsRemoteVideos diff --git a/server/middlewares/validators/remote/videos.ts b/server/middlewares/validators/remote/videos.ts new file mode 100644 index 000000000..3380c29e2 --- /dev/null +++ b/server/middlewares/validators/remote/videos.ts @@ -0,0 +1,34 @@ +import { logger } from '../../../helpers' +import { checkErrors } from '../utils' + +function remoteVideosValidator (req, res, next) { + req.checkBody('data').isEachRemoteRequestVideosValid() + + logger.debug('Checking remoteVideos parameters', { parameters: req.body }) + + checkErrors(req, res, next) +} + +function remoteQaduVideosValidator (req, res, next) { + req.checkBody('data').isEachRemoteRequestVideosQaduValid() + + logger.debug('Checking remoteQaduVideos parameters', { parameters: req.body }) + + checkErrors(req, res, next) +} + +function remoteEventsVideosValidator (req, res, next) { + req.checkBody('data').isEachRemoteRequestVideosEventsValid() + + logger.debug('Checking remoteEventsVideos parameters', { parameters: req.body }) + + checkErrors(req, res, next) +} + +// --------------------------------------------------------------------------- + +export { + remoteVideosValidator, + remoteQaduVideosValidator, + remoteEventsVideosValidator +} diff --git a/server/middlewares/validators/sort.js b/server/middlewares/validators/sort.js deleted file mode 100644 index 017d266e6..000000000 --- a/server/middlewares/validators/sort.js +++ /dev/null @@ -1,48 +0,0 @@ -'use strict' - -const checkErrors = require('./utils').checkErrors -const constants = require('../../initializers/constants') -const logger = require('../../helpers/logger') - -const validatorsSort = { - usersSort, - videoAbusesSort, - videosSort -} - -// Initialize constants here for better performances -const SORTABLE_USERS_COLUMNS = createSortableColumns(constants.SORTABLE_COLUMNS.USERS) -const SORTABLE_VIDEO_ABUSES_COLUMNS = createSortableColumns(constants.SORTABLE_COLUMNS.VIDEO_ABUSES) -const SORTABLE_VIDEOS_COLUMNS = createSortableColumns(constants.SORTABLE_COLUMNS.VIDEOS) - -function usersSort (req, res, next) { - checkSort(req, res, next, SORTABLE_USERS_COLUMNS) -} - -function videoAbusesSort (req, res, next) { - checkSort(req, res, next, SORTABLE_VIDEO_ABUSES_COLUMNS) -} - -function videosSort (req, res, next) { - checkSort(req, res, next, SORTABLE_VIDEOS_COLUMNS) -} - -// --------------------------------------------------------------------------- - -module.exports = validatorsSort - -// --------------------------------------------------------------------------- - -function checkSort (req, res, next, sortableColumns) { - req.checkQuery('sort', 'Should have correct sortable column').optional().isIn(sortableColumns) - - logger.debug('Checking sort parameters', { parameters: req.query }) - - checkErrors(req, res, next) -} - -function createSortableColumns (sortableColumns) { - const sortableColumnDesc = sortableColumns.map(sortableColumn => '-' + sortableColumn) - - return sortableColumns.concat(sortableColumnDesc) -} diff --git a/server/middlewares/validators/sort.ts b/server/middlewares/validators/sort.ts new file mode 100644 index 000000000..ebc7333c7 --- /dev/null +++ b/server/middlewares/validators/sort.ts @@ -0,0 +1,44 @@ +import { checkErrors } from './utils' +import { logger } from '../../helpers' +import { SORTABLE_COLUMNS } from '../../initializers' + +// Initialize constants here for better performances +const SORTABLE_USERS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.USERS) +const SORTABLE_VIDEO_ABUSES_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_ABUSES) +const SORTABLE_VIDEOS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEOS) + +function usersSortValidator (req, res, next) { + checkSort(req, res, next, SORTABLE_USERS_COLUMNS) +} + +function videoAbusesSortValidator (req, res, next) { + checkSort(req, res, next, SORTABLE_VIDEO_ABUSES_COLUMNS) +} + +function videosSortValidator (req, res, next) { + checkSort(req, res, next, SORTABLE_VIDEOS_COLUMNS) +} + +// --------------------------------------------------------------------------- + +export { + usersSortValidator, + videoAbusesSortValidator, + videosSortValidator +} + +// --------------------------------------------------------------------------- + +function checkSort (req, res, next, sortableColumns) { + req.checkQuery('sort', 'Should have correct sortable column').optional().isIn(sortableColumns) + + logger.debug('Checking sort parameters', { parameters: req.query }) + + checkErrors(req, res, next) +} + +function createSortableColumns (sortableColumns) { + const sortableColumnDesc = sortableColumns.map(sortableColumn => '-' + sortableColumn) + + return sortableColumns.concat(sortableColumnDesc) +} diff --git a/server/middlewares/validators/users.js b/server/middlewares/validators/users.js deleted file mode 100644 index 1e7a64793..000000000 --- a/server/middlewares/validators/users.js +++ /dev/null @@ -1,88 +0,0 @@ -'use strict' - -const checkErrors = require('./utils').checkErrors -const db = require('../../initializers/database') -const logger = require('../../helpers/logger') - -const validatorsUsers = { - usersAdd, - usersRemove, - usersUpdate, - usersVideoRating -} - -function usersAdd (req, res, next) { - req.checkBody('username', 'Should have a valid username').isUserUsernameValid() - req.checkBody('password', 'Should have a valid password').isUserPasswordValid() - req.checkBody('email', 'Should have a valid email').isEmail() - - logger.debug('Checking usersAdd parameters', { parameters: req.body }) - - checkErrors(req, res, function () { - db.User.loadByUsernameOrEmail(req.body.username, req.body.email, function (err, user) { - if (err) { - logger.error('Error in usersAdd request validator.', { error: err }) - return res.sendStatus(500) - } - - if (user) return res.status(409).send('User already exists.') - - next() - }) - }) -} - -function usersRemove (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isInt() - - logger.debug('Checking usersRemove parameters', { parameters: req.params }) - - checkErrors(req, res, function () { - db.User.loadById(req.params.id, function (err, user) { - if (err) { - logger.error('Error in usersRemove request validator.', { error: err }) - return res.sendStatus(500) - } - - if (!user) return res.status(404).send('User not found') - - if (user.username === 'root') return res.status(400).send('Cannot remove the root user') - - next() - }) - }) -} - -function usersUpdate (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isInt() - // Add old password verification - req.checkBody('password', 'Should have a valid password').optional().isUserPasswordValid() - req.checkBody('displayNSFW', 'Should have a valid display Not Safe For Work attribute').optional().isUserDisplayNSFWValid() - - logger.debug('Checking usersUpdate parameters', { parameters: req.body }) - - checkErrors(req, res, next) -} - -function usersVideoRating (req, res, next) { - req.checkParams('videoId', 'Should have a valid video id').notEmpty().isUUID(4) - - logger.debug('Checking usersVideoRating parameters', { parameters: req.params }) - - checkErrors(req, res, function () { - db.Video.load(req.params.videoId, function (err, video) { - if (err) { - logger.error('Error in user request validator.', { error: err }) - return res.sendStatus(500) - } - - if (!video) return res.status(404).send('Video not found') - - next() - }) - }) -} - -// --------------------------------------------------------------------------- - -module.exports = validatorsUsers diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts new file mode 100644 index 000000000..a9149fe1b --- /dev/null +++ b/server/middlewares/validators/users.ts @@ -0,0 +1,84 @@ +const db = require('../../initializers/database') +import { checkErrors } from './utils' +import { logger } from '../../helpers' + +function usersAddValidator (req, res, next) { + req.checkBody('username', 'Should have a valid username').isUserUsernameValid() + req.checkBody('password', 'Should have a valid password').isUserPasswordValid() + req.checkBody('email', 'Should have a valid email').isEmail() + + logger.debug('Checking usersAdd parameters', { parameters: req.body }) + + checkErrors(req, res, function () { + db.User.loadByUsernameOrEmail(req.body.username, req.body.email, function (err, user) { + if (err) { + logger.error('Error in usersAdd request validator.', { error: err }) + return res.sendStatus(500) + } + + if (user) return res.status(409).send('User already exists.') + + next() + }) + }) +} + +function usersRemoveValidator (req, res, next) { + req.checkParams('id', 'Should have a valid id').notEmpty().isInt() + + logger.debug('Checking usersRemove parameters', { parameters: req.params }) + + checkErrors(req, res, function () { + db.User.loadById(req.params.id, function (err, user) { + if (err) { + logger.error('Error in usersRemove request validator.', { error: err }) + return res.sendStatus(500) + } + + if (!user) return res.status(404).send('User not found') + + if (user.username === 'root') return res.status(400).send('Cannot remove the root user') + + next() + }) + }) +} + +function usersUpdateValidator (req, res, next) { + req.checkParams('id', 'Should have a valid id').notEmpty().isInt() + // Add old password verification + req.checkBody('password', 'Should have a valid password').optional().isUserPasswordValid() + req.checkBody('displayNSFW', 'Should have a valid display Not Safe For Work attribute').optional().isUserDisplayNSFWValid() + + logger.debug('Checking usersUpdate parameters', { parameters: req.body }) + + checkErrors(req, res, next) +} + +function usersVideoRatingValidator (req, res, next) { + req.checkParams('videoId', 'Should have a valid video id').notEmpty().isUUID(4) + + logger.debug('Checking usersVideoRating parameters', { parameters: req.params }) + + checkErrors(req, res, function () { + db.Video.load(req.params.videoId, function (err, video) { + if (err) { + logger.error('Error in user request validator.', { error: err }) + return res.sendStatus(500) + } + + if (!video) return res.status(404).send('Video not found') + + next() + }) + }) +} + +// --------------------------------------------------------------------------- + +export { + usersAddValidator, + usersRemoveValidator, + usersUpdateValidator, + usersVideoRatingValidator +} diff --git a/server/middlewares/validators/utils.js b/server/middlewares/validators/utils.js deleted file mode 100644 index 3741b84c6..000000000 --- a/server/middlewares/validators/utils.js +++ /dev/null @@ -1,25 +0,0 @@ -'use strict' - -const util = require('util') - -const logger = require('../../helpers/logger') - -const validatorsUtils = { - checkErrors -} - -function checkErrors (req, res, next, statusCode) { - if (statusCode === undefined) statusCode = 400 - const errors = req.validationErrors() - - if (errors) { - logger.warn('Incorrect request parameters', { path: req.originalUrl, err: errors }) - return res.status(statusCode).send('There have been validation errors: ' + util.inspect(errors)) - } - - return next() -} - -// --------------------------------------------------------------------------- - -module.exports = validatorsUtils diff --git a/server/middlewares/validators/utils.ts b/server/middlewares/validators/utils.ts new file mode 100644 index 000000000..710e65529 --- /dev/null +++ b/server/middlewares/validators/utils.ts @@ -0,0 +1,21 @@ +import { inspect } from 'util' + +import { logger } from '../../helpers' + +function checkErrors (req, res, next, statusCode?) { + if (statusCode === undefined) statusCode = 400 + const errors = req.validationErrors() + + if (errors) { + logger.warn('Incorrect request parameters', { path: req.originalUrl, err: errors }) + return res.status(statusCode).send('There have been validation errors: ' + inspect(errors)) + } + + return next() +} + +// --------------------------------------------------------------------------- + +export { + checkErrors +} diff --git a/server/middlewares/validators/videos.js b/server/middlewares/validators/videos.js deleted file mode 100644 index f18ca1597..000000000 --- a/server/middlewares/validators/videos.js +++ /dev/null @@ -1,204 +0,0 @@ -'use strict' - -const checkErrors = require('./utils').checkErrors -const constants = require('../../initializers/constants') -const customVideosValidators = require('../../helpers/custom-validators').videos -const db = require('../../initializers/database') -const logger = require('../../helpers/logger') - -const validatorsVideos = { - videosAdd, - videosUpdate, - videosGet, - videosRemove, - videosSearch, - - videoAbuseReport, - - videoRate, - - videosBlacklist -} - -function videosAdd (req, res, next) { - req.checkBody('videofile', 'Should have a valid file').isVideoFile(req.files) - req.checkBody('name', 'Should have a valid name').isVideoNameValid() - req.checkBody('category', 'Should have a valid category').isVideoCategoryValid() - req.checkBody('licence', 'Should have a valid licence').isVideoLicenceValid() - req.checkBody('language', 'Should have a valid language').optional().isVideoLanguageValid() - req.checkBody('nsfw', 'Should have a valid NSFW attribute').isVideoNSFWValid() - req.checkBody('description', 'Should have a valid description').isVideoDescriptionValid() - req.checkBody('tags', 'Should have correct tags').optional().isVideoTagsValid() - - logger.debug('Checking videosAdd parameters', { parameters: req.body, files: req.files }) - - checkErrors(req, res, function () { - const videoFile = req.files.videofile[0] - - db.Video.getDurationFromFile(videoFile.path, function (err, duration) { - if (err) { - return res.status(400).send('Cannot retrieve metadata of the file.') - } - - if (!customVideosValidators.isVideoDurationValid(duration)) { - return res.status(400).send('Duration of the video file is too big (max: ' + constants.CONSTRAINTS_FIELDS.VIDEOS.DURATION.max + 's).') - } - - videoFile.duration = duration - next() - }) - }) -} - -function videosUpdate (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) - req.checkBody('name', 'Should have a valid name').optional().isVideoNameValid() - req.checkBody('category', 'Should have a valid category').optional().isVideoCategoryValid() - req.checkBody('licence', 'Should have a valid licence').optional().isVideoLicenceValid() - req.checkBody('language', 'Should have a valid language').optional().isVideoLanguageValid() - req.checkBody('nsfw', 'Should have a valid NSFW attribute').optional().isVideoNSFWValid() - req.checkBody('description', 'Should have a valid description').optional().isVideoDescriptionValid() - req.checkBody('tags', 'Should have correct tags').optional().isVideoTagsValid() - - logger.debug('Checking videosUpdate parameters', { parameters: req.body }) - - checkErrors(req, res, function () { - checkVideoExists(req.params.id, res, function () { - // We need to make additional checks - if (res.locals.video.isOwned() === false) { - return res.status(403).send('Cannot update video of another pod') - } - - if (res.locals.video.Author.userId !== res.locals.oauth.token.User.id) { - return res.status(403).send('Cannot update video of another user') - } - - next() - }) - }) -} - -function videosGet (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) - - logger.debug('Checking videosGet parameters', { parameters: req.params }) - - checkErrors(req, res, function () { - checkVideoExists(req.params.id, res, next) - }) -} - -function videosRemove (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) - - logger.debug('Checking videosRemove parameters', { parameters: req.params }) - - checkErrors(req, res, function () { - checkVideoExists(req.params.id, res, function () { - // We need to make additional checks - - // Check if the user who did the request is able to delete the video - checkUserCanDeleteVideo(res.locals.oauth.token.User.id, res, function () { - next() - }) - }) - }) -} - -function videosSearch (req, res, next) { - const searchableColumns = constants.SEARCHABLE_COLUMNS.VIDEOS - req.checkParams('value', 'Should have a valid search').notEmpty() - req.checkQuery('field', 'Should have correct searchable column').optional().isIn(searchableColumns) - - logger.debug('Checking videosSearch parameters', { parameters: req.params }) - - checkErrors(req, res, next) -} - -function videoAbuseReport (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) - req.checkBody('reason', 'Should have a valid reason').isVideoAbuseReasonValid() - - logger.debug('Checking videoAbuseReport parameters', { parameters: req.body }) - - checkErrors(req, res, function () { - checkVideoExists(req.params.id, res, next) - }) -} - -function videoRate (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) - req.checkBody('rating', 'Should have a valid rate type').isVideoRatingTypeValid() - - logger.debug('Checking videoRate parameters', { parameters: req.body }) - - checkErrors(req, res, function () { - checkVideoExists(req.params.id, res, next) - }) -} - -function videosBlacklist (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) - - logger.debug('Checking videosBlacklist parameters', { parameters: req.params }) - - checkErrors(req, res, function () { - checkVideoExists(req.params.id, res, function () { - checkVideoIsBlacklistable(req, res, next) - }) - }) -} - -// --------------------------------------------------------------------------- - -module.exports = validatorsVideos - -// --------------------------------------------------------------------------- - -function checkVideoExists (id, res, callback) { - db.Video.loadAndPopulateAuthorAndPodAndTags(id, function (err, video) { - if (err) { - logger.error('Error in video request validator.', { error: err }) - return res.sendStatus(500) - } - - if (!video) return res.status(404).send('Video not found') - - res.locals.video = video - callback() - }) -} - -function checkUserCanDeleteVideo (userId, res, callback) { - // Retrieve the user who did the request - db.User.loadById(userId, function (err, user) { - if (err) { - logger.error('Error in video request validator.', { error: err }) - return res.sendStatus(500) - } - - // Check if the user can delete the video - // The user can delete it if s/he is an admin - // Or if s/he is the video's author - if (user.isAdmin() === false) { - if (res.locals.video.isOwned() === false) { - return res.status(403).send('Cannot remove video of another pod') - } - - if (res.locals.video.Author.userId !== res.locals.oauth.token.User.id) { - return res.status(403).send('Cannot remove video of another user') - } - } - - // If we reach this comment, we can delete the video - callback() - }) -} - -function checkVideoIsBlacklistable (req, res, callback) { - if (res.locals.video.isOwned() === true) { - return res.status(403).send('Cannot blacklist a local video') - } - - callback() -} diff --git a/server/middlewares/validators/videos.ts b/server/middlewares/validators/videos.ts new file mode 100644 index 000000000..5a49cf73c --- /dev/null +++ b/server/middlewares/validators/videos.ts @@ -0,0 +1,199 @@ +const db = require('../../initializers/database') +import { checkErrors } from './utils' +import { CONSTRAINTS_FIELDS, SEARCHABLE_COLUMNS } from '../../initializers' +import { logger, isVideoDurationValid } from '../../helpers' + +function videosAddValidator (req, res, next) { + req.checkBody('videofile', 'Should have a valid file').isVideoFile(req.files) + req.checkBody('name', 'Should have a valid name').isVideoNameValid() + req.checkBody('category', 'Should have a valid category').isVideoCategoryValid() + req.checkBody('licence', 'Should have a valid licence').isVideoLicenceValid() + req.checkBody('language', 'Should have a valid language').optional().isVideoLanguageValid() + req.checkBody('nsfw', 'Should have a valid NSFW attribute').isVideoNSFWValid() + req.checkBody('description', 'Should have a valid description').isVideoDescriptionValid() + req.checkBody('tags', 'Should have correct tags').optional().isVideoTagsValid() + + logger.debug('Checking videosAdd parameters', { parameters: req.body, files: req.files }) + + checkErrors(req, res, function () { + const videoFile = req.files.videofile[0] + + db.Video.getDurationFromFile(videoFile.path, function (err, duration) { + if (err) { + return res.status(400).send('Cannot retrieve metadata of the file.') + } + + if (!isVideoDurationValid(duration)) { + return res.status(400).send('Duration of the video file is too big (max: ' + CONSTRAINTS_FIELDS.VIDEOS.DURATION.max + 's).') + } + + videoFile.duration = duration + next() + }) + }) +} + +function videosUpdateValidator (req, res, next) { + req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) + req.checkBody('name', 'Should have a valid name').optional().isVideoNameValid() + req.checkBody('category', 'Should have a valid category').optional().isVideoCategoryValid() + req.checkBody('licence', 'Should have a valid licence').optional().isVideoLicenceValid() + req.checkBody('language', 'Should have a valid language').optional().isVideoLanguageValid() + req.checkBody('nsfw', 'Should have a valid NSFW attribute').optional().isVideoNSFWValid() + req.checkBody('description', 'Should have a valid description').optional().isVideoDescriptionValid() + req.checkBody('tags', 'Should have correct tags').optional().isVideoTagsValid() + + logger.debug('Checking videosUpdate parameters', { parameters: req.body }) + + checkErrors(req, res, function () { + checkVideoExists(req.params.id, res, function () { + // We need to make additional checks + if (res.locals.video.isOwned() === false) { + return res.status(403).send('Cannot update video of another pod') + } + + if (res.locals.video.Author.userId !== res.locals.oauth.token.User.id) { + return res.status(403).send('Cannot update video of another user') + } + + next() + }) + }) +} + +function videosGetValidator (req, res, next) { + req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) + + logger.debug('Checking videosGet parameters', { parameters: req.params }) + + checkErrors(req, res, function () { + checkVideoExists(req.params.id, res, next) + }) +} + +function videosRemoveValidator (req, res, next) { + req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) + + logger.debug('Checking videosRemove parameters', { parameters: req.params }) + + checkErrors(req, res, function () { + checkVideoExists(req.params.id, res, function () { + // We need to make additional checks + + // Check if the user who did the request is able to delete the video + checkUserCanDeleteVideo(res.locals.oauth.token.User.id, res, function () { + next() + }) + }) + }) +} + +function videosSearchValidator (req, res, next) { + const searchableColumns = SEARCHABLE_COLUMNS.VIDEOS + req.checkParams('value', 'Should have a valid search').notEmpty() + req.checkQuery('field', 'Should have correct searchable column').optional().isIn(searchableColumns) + + logger.debug('Checking videosSearch parameters', { parameters: req.params }) + + checkErrors(req, res, next) +} + +function videoAbuseReportValidator (req, res, next) { + req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) + req.checkBody('reason', 'Should have a valid reason').isVideoAbuseReasonValid() + + logger.debug('Checking videoAbuseReport parameters', { parameters: req.body }) + + checkErrors(req, res, function () { + checkVideoExists(req.params.id, res, next) + }) +} + +function videoRateValidator (req, res, next) { + req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) + req.checkBody('rating', 'Should have a valid rate type').isVideoRatingTypeValid() + + logger.debug('Checking videoRate parameters', { parameters: req.body }) + + checkErrors(req, res, function () { + checkVideoExists(req.params.id, res, next) + }) +} + +function videosBlacklistValidator (req, res, next) { + req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) + + logger.debug('Checking videosBlacklist parameters', { parameters: req.params }) + + checkErrors(req, res, function () { + checkVideoExists(req.params.id, res, function () { + checkVideoIsBlacklistable(req, res, next) + }) + }) +} + +// --------------------------------------------------------------------------- + +export { + videosAddValidator, + videosUpdateValidator, + videosGetValidator, + videosRemoveValidator, + videosSearchValidator, + + videoAbuseReportValidator, + + videoRateValidator, + + videosBlacklistValidator +} + +// --------------------------------------------------------------------------- + +function checkVideoExists (id, res, callback) { + db.Video.loadAndPopulateAuthorAndPodAndTags(id, function (err, video) { + if (err) { + logger.error('Error in video request validator.', { error: err }) + return res.sendStatus(500) + } + + if (!video) return res.status(404).send('Video not found') + + res.locals.video = video + callback() + }) +} + +function checkUserCanDeleteVideo (userId, res, callback) { + // Retrieve the user who did the request + db.User.loadById(userId, function (err, user) { + if (err) { + logger.error('Error in video request validator.', { error: err }) + return res.sendStatus(500) + } + + // Check if the user can delete the video + // The user can delete it if s/he is an admin + // Or if s/he is the video's author + if (user.isAdmin() === false) { + if (res.locals.video.isOwned() === false) { + return res.status(403).send('Cannot remove video of another pod') + } + + if (res.locals.video.Author.userId !== res.locals.oauth.token.User.id) { + return res.status(403).send('Cannot remove video of another user') + } + } + + // If we reach this comment, we can delete the video + callback() + }) +} + +function checkVideoIsBlacklistable (req, res, callback) { + if (res.locals.video.isOwned() === true) { + return res.status(403).send('Cannot blacklist a local video') + } + + callback() +} diff --git a/server/models/application.js b/server/models/application.js deleted file mode 100644 index 64e1a0540..000000000 --- a/server/models/application.js +++ /dev/null @@ -1,52 +0,0 @@ -'use strict' - -module.exports = function (sequelize, DataTypes) { - const Application = sequelize.define('Application', - { - migrationVersion: { - type: DataTypes.INTEGER, - defaultValue: 0, - allowNull: false, - validate: { - isInt: true - } - } - }, - { - classMethods: { - loadMigrationVersion, - updateMigrationVersion - } - } - ) - - return Application -} - -// --------------------------------------------------------------------------- - -function loadMigrationVersion (callback) { - const query = { - attributes: [ 'migrationVersion' ] - } - - return this.findOne(query).asCallback(function (err, data) { - const version = data ? data.migrationVersion : null - - return callback(err, version) - }) -} - -function updateMigrationVersion (newVersion, transaction, callback) { - const options = { - where: {} - } - - if (!callback) { - transaction = callback - } else { - options.transaction = transaction - } - - return this.update({ migrationVersion: newVersion }, options).asCallback(callback) -} diff --git a/server/models/application.ts b/server/models/application.ts new file mode 100644 index 000000000..38a57e327 --- /dev/null +++ b/server/models/application.ts @@ -0,0 +1,50 @@ +module.exports = function (sequelize, DataTypes) { + const Application = sequelize.define('Application', + { + migrationVersion: { + type: DataTypes.INTEGER, + defaultValue: 0, + allowNull: false, + validate: { + isInt: true + } + } + }, + { + classMethods: { + loadMigrationVersion, + updateMigrationVersion + } + } + ) + + return Application +} + +// --------------------------------------------------------------------------- + +function loadMigrationVersion (callback) { + const query = { + attributes: [ 'migrationVersion' ] + } + + return this.findOne(query).asCallback(function (err, data) { + const version = data ? data.migrationVersion : null + + return callback(err, version) + }) +} + +function updateMigrationVersion (newVersion, transaction, callback) { + const options: { where?: any, transaction?: any } = { + where: {} + } + + if (!callback) { + transaction = callback + } else { + options.transaction = transaction + } + + return this.update({ migrationVersion: newVersion }, options).asCallback(callback) +} diff --git a/server/models/author.js b/server/models/author.js deleted file mode 100644 index 34b013097..000000000 --- a/server/models/author.js +++ /dev/null @@ -1,92 +0,0 @@ -'use strict' - -const customUsersValidators = require('../helpers/custom-validators').users - -module.exports = function (sequelize, DataTypes) { - const Author = sequelize.define('Author', - { - name: { - type: DataTypes.STRING, - allowNull: false, - validate: { - usernameValid: function (value) { - const res = customUsersValidators.isUserUsernameValid(value) - if (res === false) throw new Error('Username is not valid.') - } - } - } - }, - { - indexes: [ - { - fields: [ 'name' ] - }, - { - fields: [ 'podId' ] - }, - { - fields: [ 'userId' ], - unique: true - }, - { - fields: [ 'name', 'podId' ], - unique: true - } - ], - classMethods: { - associate, - - findOrCreateAuthor - } - } - ) - - return Author -} - -// --------------------------------------------------------------------------- - -function associate (models) { - this.belongsTo(models.Pod, { - foreignKey: { - name: 'podId', - allowNull: true - }, - onDelete: 'cascade' - }) - - this.belongsTo(models.User, { - foreignKey: { - name: 'userId', - allowNull: true - }, - onDelete: 'cascade' - }) -} - -function findOrCreateAuthor (name, podId, userId, transaction, callback) { - if (!callback) { - callback = transaction - transaction = null - } - - const author = { - name, - podId, - userId - } - - const query = { - where: author, - defaults: author - } - - if (transaction) query.transaction = transaction - - this.findOrCreate(query).asCallback(function (err, result) { - if (err) return callback(err) - - // [ instance, wasCreated ] - return callback(null, result[0]) - }) -} diff --git a/server/models/author.ts b/server/models/author.ts new file mode 100644 index 000000000..4a7396929 --- /dev/null +++ b/server/models/author.ts @@ -0,0 +1,90 @@ +import { isUserUsernameValid } from '../helpers' + +module.exports = function (sequelize, DataTypes) { + const Author = sequelize.define('Author', + { + name: { + type: DataTypes.STRING, + allowNull: false, + validate: { + usernameValid: function (value) { + const res = isUserUsernameValid(value) + if (res === false) throw new Error('Username is not valid.') + } + } + } + }, + { + indexes: [ + { + fields: [ 'name' ] + }, + { + fields: [ 'podId' ] + }, + { + fields: [ 'userId' ], + unique: true + }, + { + fields: [ 'name', 'podId' ], + unique: true + } + ], + classMethods: { + associate, + + findOrCreateAuthor + } + } + ) + + return Author +} + +// --------------------------------------------------------------------------- + +function associate (models) { + this.belongsTo(models.Pod, { + foreignKey: { + name: 'podId', + allowNull: true + }, + onDelete: 'cascade' + }) + + this.belongsTo(models.User, { + foreignKey: { + name: 'userId', + allowNull: true + }, + onDelete: 'cascade' + }) +} + +function findOrCreateAuthor (name, podId, userId, transaction, callback) { + if (!callback) { + callback = transaction + transaction = null + } + + const author = { + name, + podId, + userId + } + + const query: any = { + where: author, + defaults: author + } + + if (transaction) query.transaction = transaction + + this.findOrCreate(query).asCallback(function (err, result) { + if (err) return callback(err) + + // [ instance, wasCreated ] + return callback(null, result[0]) + }) +} diff --git a/server/models/job.js b/server/models/job.js deleted file mode 100644 index 949f88d44..000000000 --- a/server/models/job.js +++ /dev/null @@ -1,54 +0,0 @@ -'use strict' - -const values = require('lodash/values') - -const constants = require('../initializers/constants') - -// --------------------------------------------------------------------------- - -module.exports = function (sequelize, DataTypes) { - const Job = sequelize.define('Job', - { - state: { - type: DataTypes.ENUM(values(constants.JOB_STATES)), - allowNull: false - }, - handlerName: { - type: DataTypes.STRING, - allowNull: false - }, - handlerInputData: { - type: DataTypes.JSON, - allowNull: true - } - }, - { - indexes: [ - { - fields: [ 'state' ] - } - ], - classMethods: { - listWithLimit - } - } - ) - - return Job -} - -// --------------------------------------------------------------------------- - -function listWithLimit (limit, state, callback) { - const query = { - order: [ - [ 'id', 'ASC' ] - ], - limit: limit, - where: { - state - } - } - - return this.findAll(query).asCallback(callback) -} diff --git a/server/models/job.ts b/server/models/job.ts new file mode 100644 index 000000000..6843e399b --- /dev/null +++ b/server/models/job.ts @@ -0,0 +1,52 @@ +import { values } from 'lodash' + +import { JOB_STATES } from '../initializers' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const Job = sequelize.define('Job', + { + state: { + type: DataTypes.ENUM(values(JOB_STATES)), + allowNull: false + }, + handlerName: { + type: DataTypes.STRING, + allowNull: false + }, + handlerInputData: { + type: DataTypes.JSON, + allowNull: true + } + }, + { + indexes: [ + { + fields: [ 'state' ] + } + ], + classMethods: { + listWithLimit + } + } + ) + + return Job +} + +// --------------------------------------------------------------------------- + +function listWithLimit (limit, state, callback) { + const query = { + order: [ + [ 'id', 'ASC' ] + ], + limit: limit, + where: { + state + } + } + + return this.findAll(query).asCallback(callback) +} diff --git a/server/models/oauth-client.js b/server/models/oauth-client.js deleted file mode 100644 index 021a34007..000000000 --- a/server/models/oauth-client.js +++ /dev/null @@ -1,62 +0,0 @@ -'use strict' - -module.exports = function (sequelize, DataTypes) { - const OAuthClient = sequelize.define('OAuthClient', - { - clientId: { - type: DataTypes.STRING, - allowNull: false - }, - clientSecret: { - type: DataTypes.STRING, - allowNull: false - }, - grants: { - type: DataTypes.ARRAY(DataTypes.STRING) - }, - redirectUris: { - type: DataTypes.ARRAY(DataTypes.STRING) - } - }, - { - indexes: [ - { - fields: [ 'clientId' ], - unique: true - }, - { - fields: [ 'clientId', 'clientSecret' ], - unique: true - } - ], - classMethods: { - countTotal, - getByIdAndSecret, - loadFirstClient - } - } - ) - - return OAuthClient -} - -// --------------------------------------------------------------------------- - -function countTotal (callback) { - return this.count().asCallback(callback) -} - -function loadFirstClient (callback) { - return this.findOne().asCallback(callback) -} - -function getByIdAndSecret (clientId, clientSecret) { - const query = { - where: { - clientId: clientId, - clientSecret: clientSecret - } - } - - return this.findOne(query) -} diff --git a/server/models/oauth-client.ts b/server/models/oauth-client.ts new file mode 100644 index 000000000..3198a85ef --- /dev/null +++ b/server/models/oauth-client.ts @@ -0,0 +1,60 @@ +module.exports = function (sequelize, DataTypes) { + const OAuthClient = sequelize.define('OAuthClient', + { + clientId: { + type: DataTypes.STRING, + allowNull: false + }, + clientSecret: { + type: DataTypes.STRING, + allowNull: false + }, + grants: { + type: DataTypes.ARRAY(DataTypes.STRING) + }, + redirectUris: { + type: DataTypes.ARRAY(DataTypes.STRING) + } + }, + { + indexes: [ + { + fields: [ 'clientId' ], + unique: true + }, + { + fields: [ 'clientId', 'clientSecret' ], + unique: true + } + ], + classMethods: { + countTotal, + getByIdAndSecret, + loadFirstClient + } + } + ) + + return OAuthClient +} + +// --------------------------------------------------------------------------- + +function countTotal (callback) { + return this.count().asCallback(callback) +} + +function loadFirstClient (callback) { + return this.findOne().asCallback(callback) +} + +function getByIdAndSecret (clientId, clientSecret) { + const query = { + where: { + clientId: clientId, + clientSecret: clientSecret + } + } + + return this.findOne(query) +} diff --git a/server/models/oauth-token.js b/server/models/oauth-token.js deleted file mode 100644 index 68e7c9ff7..000000000 --- a/server/models/oauth-token.js +++ /dev/null @@ -1,144 +0,0 @@ -'use strict' - -const logger = require('../helpers/logger') - -// --------------------------------------------------------------------------- - -module.exports = function (sequelize, DataTypes) { - const OAuthToken = sequelize.define('OAuthToken', - { - accessToken: { - type: DataTypes.STRING, - allowNull: false - }, - accessTokenExpiresAt: { - type: DataTypes.DATE, - allowNull: false - }, - refreshToken: { - type: DataTypes.STRING, - allowNull: false - }, - refreshTokenExpiresAt: { - type: DataTypes.DATE, - allowNull: false - } - }, - { - indexes: [ - { - fields: [ 'refreshToken' ], - unique: true - }, - { - fields: [ 'accessToken' ], - unique: true - }, - { - fields: [ 'userId' ] - }, - { - fields: [ 'oAuthClientId' ] - } - ], - classMethods: { - associate, - - getByRefreshTokenAndPopulateClient, - getByTokenAndPopulateUser, - getByRefreshTokenAndPopulateUser, - removeByUserId - } - } - ) - - return OAuthToken -} - -// --------------------------------------------------------------------------- - -function associate (models) { - this.belongsTo(models.User, { - foreignKey: { - name: 'userId', - allowNull: false - }, - onDelete: 'cascade' - }) - - this.belongsTo(models.OAuthClient, { - foreignKey: { - name: 'oAuthClientId', - allowNull: false - }, - onDelete: 'cascade' - }) -} - -function getByRefreshTokenAndPopulateClient (refreshToken) { - const query = { - where: { - refreshToken: refreshToken - }, - include: [ this.associations.OAuthClient ] - } - - return this.findOne(query).then(function (token) { - if (!token) return token - - const tokenInfos = { - refreshToken: token.refreshToken, - refreshTokenExpiresAt: token.refreshTokenExpiresAt, - client: { - id: token.client.id - }, - user: { - id: token.user - } - } - - return tokenInfos - }).catch(function (err) { - logger.info('getRefreshToken error.', { error: err }) - }) -} - -function getByTokenAndPopulateUser (bearerToken) { - const query = { - where: { - accessToken: bearerToken - }, - include: [ this.sequelize.models.User ] - } - - return this.findOne(query).then(function (token) { - if (token) token.user = token.User - - return token - }) -} - -function getByRefreshTokenAndPopulateUser (refreshToken) { - const query = { - where: { - refreshToken: refreshToken - }, - include: [ this.sequelize.models.User ] - } - - return this.findOne(query).then(function (token) { - token.user = token.User - - return token - }) -} - -function removeByUserId (userId, callback) { - const query = { - where: { - userId: userId - } - } - - return this.destroy(query).asCallback(callback) -} diff --git a/server/models/oauth-token.ts b/server/models/oauth-token.ts new file mode 100644 index 000000000..74c9180eb --- /dev/null +++ b/server/models/oauth-token.ts @@ -0,0 +1,142 @@ +import { logger } from '../helpers' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const OAuthToken = sequelize.define('OAuthToken', + { + accessToken: { + type: DataTypes.STRING, + allowNull: false + }, + accessTokenExpiresAt: { + type: DataTypes.DATE, + allowNull: false + }, + refreshToken: { + type: DataTypes.STRING, + allowNull: false + }, + refreshTokenExpiresAt: { + type: DataTypes.DATE, + allowNull: false + } + }, + { + indexes: [ + { + fields: [ 'refreshToken' ], + unique: true + }, + { + fields: [ 'accessToken' ], + unique: true + }, + { + fields: [ 'userId' ] + }, + { + fields: [ 'oAuthClientId' ] + } + ], + classMethods: { + associate, + + getByRefreshTokenAndPopulateClient, + getByTokenAndPopulateUser, + getByRefreshTokenAndPopulateUser, + removeByUserId + } + } + ) + + return OAuthToken +} + +// --------------------------------------------------------------------------- + +function associate (models) { + this.belongsTo(models.User, { + foreignKey: { + name: 'userId', + allowNull: false + }, + onDelete: 'cascade' + }) + + this.belongsTo(models.OAuthClient, { + foreignKey: { + name: 'oAuthClientId', + allowNull: false + }, + onDelete: 'cascade' + }) +} + +function getByRefreshTokenAndPopulateClient (refreshToken) { + const query = { + where: { + refreshToken: refreshToken + }, + include: [ this.associations.OAuthClient ] + } + + return this.findOne(query).then(function (token) { + if (!token) return token + + const tokenInfos = { + refreshToken: token.refreshToken, + refreshTokenExpiresAt: token.refreshTokenExpiresAt, + client: { + id: token.client.id + }, + user: { + id: token.user + } + } + + return tokenInfos + }).catch(function (err) { + logger.info('getRefreshToken error.', { error: err }) + }) +} + +function getByTokenAndPopulateUser (bearerToken) { + const query = { + where: { + accessToken: bearerToken + }, + include: [ this.sequelize.models.User ] + } + + return this.findOne(query).then(function (token) { + if (token) token.user = token.User + + return token + }) +} + +function getByRefreshTokenAndPopulateUser (refreshToken) { + const query = { + where: { + refreshToken: refreshToken + }, + include: [ this.sequelize.models.User ] + } + + return this.findOne(query).then(function (token) { + token.user = token.User + + return token + }) +} + +function removeByUserId (userId, callback) { + const query = { + where: { + userId: userId + } + } + + return this.destroy(query).asCallback(callback) +} diff --git a/server/models/pod.js b/server/models/pod.js deleted file mode 100644 index 8e2d488e1..000000000 --- a/server/models/pod.js +++ /dev/null @@ -1,273 +0,0 @@ -'use strict' - -const each = require('async/each') -const map = require('lodash/map') -const waterfall = require('async/waterfall') - -const constants = require('../initializers/constants') -const logger = require('../helpers/logger') -const customPodsValidators = require('../helpers/custom-validators').pods - -// --------------------------------------------------------------------------- - -module.exports = function (sequelize, DataTypes) { - const Pod = sequelize.define('Pod', - { - host: { - type: DataTypes.STRING, - allowNull: false, - validate: { - isHost: function (value) { - const res = customPodsValidators.isHostValid(value) - if (res === false) throw new Error('Host not valid.') - } - } - }, - publicKey: { - type: DataTypes.STRING(5000), - allowNull: false - }, - score: { - type: DataTypes.INTEGER, - defaultValue: constants.FRIEND_SCORE.BASE, - allowNull: false, - validate: { - isInt: true, - max: constants.FRIEND_SCORE.MAX - } - }, - email: { - type: DataTypes.STRING(400), - allowNull: false, - validate: { - isEmail: true - } - } - }, - { - indexes: [ - { - fields: [ 'host' ], - unique: true - }, - { - fields: [ 'score' ] - } - ], - classMethods: { - associate, - - countAll, - incrementScores, - list, - listAllIds, - listRandomPodIdsWithRequest, - listBadPods, - load, - loadByHost, - updatePodsScore, - removeAll - }, - instanceMethods: { - toFormatedJSON - } - } - ) - - return Pod -} - -// ------------------------------ METHODS ------------------------------ - -function toFormatedJSON () { - const json = { - id: this.id, - host: this.host, - email: this.email, - score: this.score, - createdAt: this.createdAt - } - - return json -} - -// ------------------------------ Statics ------------------------------ - -function associate (models) { - this.belongsToMany(models.Request, { - foreignKey: 'podId', - through: models.RequestToPod, - onDelete: 'cascade' - }) -} - -function countAll (callback) { - return this.count().asCallback(callback) -} - -function incrementScores (ids, value, callback) { - if (!callback) callback = function () {} - - const update = { - score: this.sequelize.literal('score +' + value) - } - - const options = { - where: { - id: { - $in: ids - } - }, - // In this case score is a literal and not an integer so we do not validate it - validate: false - } - - return this.update(update, options).asCallback(callback) -} - -function list (callback) { - return this.findAll().asCallback(callback) -} - -function listAllIds (transaction, callback) { - if (!callback) { - callback = transaction - transaction = null - } - - const query = { - attributes: [ 'id' ] - } - - if (transaction) query.transaction = transaction - - return this.findAll(query).asCallback(function (err, pods) { - if (err) return callback(err) - - return callback(null, map(pods, 'id')) - }) -} - -function listRandomPodIdsWithRequest (limit, tableWithPods, tableWithPodsJoins, callback) { - if (!callback) { - callback = tableWithPodsJoins - tableWithPodsJoins = '' - } - - const self = this - - self.count().asCallback(function (err, count) { - if (err) return callback(err) - - // Optimization... - if (count === 0) return callback(null, []) - - let start = Math.floor(Math.random() * count) - limit - if (start < 0) start = 0 - - const query = { - attributes: [ 'id' ], - order: [ - [ 'id', 'ASC' ] - ], - offset: start, - limit: limit, - where: { - id: { - $in: [ - this.sequelize.literal(`SELECT DISTINCT "${tableWithPods}"."podId" FROM "${tableWithPods}" ${tableWithPodsJoins}`) - ] - } - } - } - - return this.findAll(query).asCallback(function (err, pods) { - if (err) return callback(err) - - return callback(null, map(pods, 'id')) - }) - }) -} - -function listBadPods (callback) { - const query = { - where: { - score: { $lte: 0 } - } - } - - return this.findAll(query).asCallback(callback) -} - -function load (id, callback) { - return this.findById(id).asCallback(callback) -} - -function loadByHost (host, callback) { - const query = { - where: { - host: host - } - } - - return this.findOne(query).asCallback(callback) -} - -function removeAll (callback) { - return this.destroy().asCallback(callback) -} - -function updatePodsScore (goodPods, badPods) { - const self = this - - logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length) - - if (goodPods.length !== 0) { - this.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) { - if (err) logger.error('Cannot increment scores of good pods.', { error: err }) - }) - } - - if (badPods.length !== 0) { - this.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) { - if (err) logger.error('Cannot decrement scores of bad pods.', { error: err }) - removeBadPods.call(self) - }) - } -} - -// --------------------------------------------------------------------------- - -// Remove pods with a score of 0 (too many requests where they were unreachable) -function removeBadPods () { - const self = this - - waterfall([ - function findBadPods (callback) { - self.sequelize.models.Pod.listBadPods(function (err, pods) { - if (err) { - logger.error('Cannot find bad pods.', { error: err }) - return callback(err) - } - - return callback(null, pods) - }) - }, - - function removeTheseBadPods (pods, callback) { - each(pods, function (pod, callbackEach) { - pod.destroy().asCallback(callbackEach) - }, function (err) { - return callback(err, pods.length) - }) - } - ], function (err, numberOfPodsRemoved) { - if (err) { - logger.error('Cannot remove bad pods.', { error: err }) - } else if (numberOfPodsRemoved) { - logger.info('Removed %d pods.', numberOfPodsRemoved) - } else { - logger.info('No need to remove bad pods.') - } - }) -} diff --git a/server/models/pod.ts b/server/models/pod.ts new file mode 100644 index 000000000..0e0262978 --- /dev/null +++ b/server/models/pod.ts @@ -0,0 +1,269 @@ +import { each, waterfall } from 'async' +import { map } from 'lodash' + +import { FRIEND_SCORE, PODS_SCORE } from '../initializers' +import { logger, isHostValid } from '../helpers' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const Pod = sequelize.define('Pod', + { + host: { + type: DataTypes.STRING, + allowNull: false, + validate: { + isHost: function (value) { + const res = isHostValid(value) + if (res === false) throw new Error('Host not valid.') + } + } + }, + publicKey: { + type: DataTypes.STRING(5000), + allowNull: false + }, + score: { + type: DataTypes.INTEGER, + defaultValue: FRIEND_SCORE.BASE, + allowNull: false, + validate: { + isInt: true, + max: FRIEND_SCORE.MAX + } + }, + email: { + type: DataTypes.STRING(400), + allowNull: false, + validate: { + isEmail: true + } + } + }, + { + indexes: [ + { + fields: [ 'host' ], + unique: true + }, + { + fields: [ 'score' ] + } + ], + classMethods: { + associate, + + countAll, + incrementScores, + list, + listAllIds, + listRandomPodIdsWithRequest, + listBadPods, + load, + loadByHost, + updatePodsScore, + removeAll + }, + instanceMethods: { + toFormatedJSON + } + } + ) + + return Pod +} + +// ------------------------------ METHODS ------------------------------ + +function toFormatedJSON () { + const json = { + id: this.id, + host: this.host, + email: this.email, + score: this.score, + createdAt: this.createdAt + } + + return json +} + +// ------------------------------ Statics ------------------------------ + +function associate (models) { + this.belongsToMany(models.Request, { + foreignKey: 'podId', + through: models.RequestToPod, + onDelete: 'cascade' + }) +} + +function countAll (callback) { + return this.count().asCallback(callback) +} + +function incrementScores (ids, value, callback) { + if (!callback) callback = function () { /* empty */ } + + const update = { + score: this.sequelize.literal('score +' + value) + } + + const options = { + where: { + id: { + $in: ids + } + }, + // In this case score is a literal and not an integer so we do not validate it + validate: false + } + + return this.update(update, options).asCallback(callback) +} + +function list (callback) { + return this.findAll().asCallback(callback) +} + +function listAllIds (transaction, callback) { + if (!callback) { + callback = transaction + transaction = null + } + + const query: any = { + attributes: [ 'id' ] + } + + if (transaction) query.transaction = transaction + + return this.findAll(query).asCallback(function (err, pods) { + if (err) return callback(err) + + return callback(null, map(pods, 'id')) + }) +} + +function listRandomPodIdsWithRequest (limit, tableWithPods, tableWithPodsJoins, callback) { + if (!callback) { + callback = tableWithPodsJoins + tableWithPodsJoins = '' + } + + const self = this + + self.count().asCallback(function (err, count) { + if (err) return callback(err) + + // Optimization... + if (count === 0) return callback(null, []) + + let start = Math.floor(Math.random() * count) - limit + if (start < 0) start = 0 + + const query = { + attributes: [ 'id' ], + order: [ + [ 'id', 'ASC' ] + ], + offset: start, + limit: limit, + where: { + id: { + $in: [ + this.sequelize.literal(`SELECT DISTINCT "${tableWithPods}"."podId" FROM "${tableWithPods}" ${tableWithPodsJoins}`) + ] + } + } + } + + return this.findAll(query).asCallback(function (err, pods) { + if (err) return callback(err) + + return callback(null, map(pods, 'id')) + }) + }) +} + +function listBadPods (callback) { + const query = { + where: { + score: { $lte: 0 } + } + } + + return this.findAll(query).asCallback(callback) +} + +function load (id, callback) { + return this.findById(id).asCallback(callback) +} + +function loadByHost (host, callback) { + const query = { + where: { + host: host + } + } + + return this.findOne(query).asCallback(callback) +} + +function removeAll (callback) { + return this.destroy().asCallback(callback) +} + +function updatePodsScore (goodPods, badPods) { + const self = this + + logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length) + + if (goodPods.length !== 0) { + this.incrementScores(goodPods, PODS_SCORE.BONUS, function (err) { + if (err) logger.error('Cannot increment scores of good pods.', { error: err }) + }) + } + + if (badPods.length !== 0) { + this.incrementScores(badPods, PODS_SCORE.MALUS, function (err) { + if (err) logger.error('Cannot decrement scores of bad pods.', { error: err }) + removeBadPods.call(self) + }) + } +} + +// --------------------------------------------------------------------------- + +// Remove pods with a score of 0 (too many requests where they were unreachable) +function removeBadPods () { + const self = this + + waterfall([ + function findBadPods (callback) { + self.sequelize.models.Pod.listBadPods(function (err, pods) { + if (err) { + logger.error('Cannot find bad pods.', { error: err }) + return callback(err) + } + + return callback(null, pods) + }) + }, + + function removeTheseBadPods (pods, callback) { + each(pods, function (pod: any, callbackEach) { + pod.destroy().asCallback(callbackEach) + }, function (err) { + return callback(err, pods.length) + }) + } + ], function (err, numberOfPodsRemoved) { + if (err) { + logger.error('Cannot remove bad pods.', { error: err }) + } else if (numberOfPodsRemoved) { + logger.info('Removed %d pods.', numberOfPodsRemoved) + } else { + logger.info('No need to remove bad pods.') + } + }) +} diff --git a/server/models/request-to-pod.js b/server/models/request-to-pod.js deleted file mode 100644 index 0e01a842e..000000000 --- a/server/models/request-to-pod.js +++ /dev/null @@ -1,42 +0,0 @@ -'use strict' - -// --------------------------------------------------------------------------- - -module.exports = function (sequelize, DataTypes) { - const RequestToPod = sequelize.define('RequestToPod', {}, { - indexes: [ - { - fields: [ 'requestId' ] - }, - { - fields: [ 'podId' ] - }, - { - fields: [ 'requestId', 'podId' ], - unique: true - } - ], - classMethods: { - removeByRequestIdsAndPod - } - }) - - return RequestToPod -} - -// --------------------------------------------------------------------------- - -function removeByRequestIdsAndPod (requestsIds, podId, callback) { - if (!callback) callback = function () {} - - const query = { - where: { - requestId: { - $in: requestsIds - }, - podId: podId - } - } - - this.destroy(query).asCallback(callback) -} diff --git a/server/models/request-to-pod.ts b/server/models/request-to-pod.ts new file mode 100644 index 000000000..479202e40 --- /dev/null +++ b/server/models/request-to-pod.ts @@ -0,0 +1,38 @@ +module.exports = function (sequelize, DataTypes) { + const RequestToPod = sequelize.define('RequestToPod', {}, { + indexes: [ + { + fields: [ 'requestId' ] + }, + { + fields: [ 'podId' ] + }, + { + fields: [ 'requestId', 'podId' ], + unique: true + } + ], + classMethods: { + removeByRequestIdsAndPod + } + }) + + return RequestToPod +} + +// --------------------------------------------------------------------------- + +function removeByRequestIdsAndPod (requestsIds, podId, callback) { + if (!callback) callback = function () { /* empty */ } + + const query = { + where: { + requestId: { + $in: requestsIds + }, + podId: podId + } + } + + this.destroy(query).asCallback(callback) +} diff --git a/server/models/request-video-event.js b/server/models/request-video-event.js deleted file mode 100644 index 9ebeaec90..000000000 --- a/server/models/request-video-event.js +++ /dev/null @@ -1,172 +0,0 @@ -'use strict' - -/* - Request Video events (likes, dislikes, views...) -*/ - -const values = require('lodash/values') - -const constants = require('../initializers/constants') -const customVideosValidators = require('../helpers/custom-validators').videos - -// --------------------------------------------------------------------------- - -module.exports = function (sequelize, DataTypes) { - const RequestVideoEvent = sequelize.define('RequestVideoEvent', - { - type: { - type: DataTypes.ENUM(values(constants.REQUEST_VIDEO_EVENT_TYPES)), - allowNull: false - }, - count: { - type: DataTypes.INTEGER, - allowNull: false, - validate: { - countValid: function (value) { - const res = customVideosValidators.isVideoEventCountValid(value) - if (res === false) throw new Error('Video event count is not valid.') - } - } - } - }, - { - updatedAt: false, - indexes: [ - { - fields: [ 'videoId' ] - } - ], - classMethods: { - associate, - - listWithLimitAndRandom, - - countTotalRequests, - removeAll, - removeByRequestIdsAndPod - } - } - ) - - return RequestVideoEvent -} - -// ------------------------------ STATICS ------------------------------ - -function associate (models) { - this.belongsTo(models.Video, { - foreignKey: { - name: 'videoId', - allowNull: false - }, - onDelete: 'CASCADE' - }) -} - -function countTotalRequests (callback) { - const query = {} - return this.count(query).asCallback(callback) -} - -function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) { - const self = this - const Pod = this.sequelize.models.Pod - - // We make a join between videos and authors to find the podId of our video event requests - const podJoins = 'INNER JOIN "Videos" ON "Videos"."authorId" = "Authors"."id" ' + - 'INNER JOIN "RequestVideoEvents" ON "RequestVideoEvents"."videoId" = "Videos"."id"' - - Pod.listRandomPodIdsWithRequest(limitPods, 'Authors', podJoins, function (err, podIds) { - if (err) return callback(err) - - // We don't have friends that have requests - if (podIds.length === 0) return callback(null, []) - - const query = { - order: [ - [ 'id', 'ASC' ] - ], - include: [ - { - model: self.sequelize.models.Video, - include: [ - { - model: self.sequelize.models.Author, - include: [ - { - model: self.sequelize.models.Pod, - where: { - id: { - $in: podIds - } - } - } - ] - } - ] - } - ] - } - - self.findAll(query).asCallback(function (err, requests) { - if (err) return callback(err) - - const requestsGrouped = groupAndTruncateRequests(requests, limitRequestsPerPod) - return callback(err, requestsGrouped) - }) - }) -} - -function removeByRequestIdsAndPod (ids, podId, callback) { - const query = { - where: { - id: { - $in: ids - } - }, - include: [ - { - model: this.sequelize.models.Video, - include: [ - { - model: this.sequelize.models.Author, - where: { - podId - } - } - ] - } - ] - } - - this.destroy(query).asCallback(callback) -} - -function removeAll (callback) { - // Delete all requests - this.truncate({ cascade: true }).asCallback(callback) -} - -// --------------------------------------------------------------------------- - -function groupAndTruncateRequests (events, limitRequestsPerPod) { - const eventsGrouped = {} - - events.forEach(function (event) { - const pod = event.Video.Author.Pod - - if (!eventsGrouped[pod.id]) eventsGrouped[pod.id] = [] - - if (eventsGrouped[pod.id].length < limitRequestsPerPod) { - eventsGrouped[pod.id].push({ - id: event.id, - type: event.type, - count: event.count, - video: event.Video, - pod - }) - } - }) - - return eventsGrouped -} diff --git a/server/models/request-video-event.ts b/server/models/request-video-event.ts new file mode 100644 index 000000000..c61525029 --- /dev/null +++ b/server/models/request-video-event.ts @@ -0,0 +1,170 @@ +/* + Request Video events (likes, dislikes, views...) +*/ + +import { values } from 'lodash' + +import { REQUEST_VIDEO_EVENT_TYPES } from '../initializers' +import { isVideoEventCountValid } from '../helpers' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const RequestVideoEvent = sequelize.define('RequestVideoEvent', + { + type: { + type: DataTypes.ENUM(values(REQUEST_VIDEO_EVENT_TYPES)), + allowNull: false + }, + count: { + type: DataTypes.INTEGER, + allowNull: false, + validate: { + countValid: function (value) { + const res = isVideoEventCountValid(value) + if (res === false) throw new Error('Video event count is not valid.') + } + } + } + }, + { + updatedAt: false, + indexes: [ + { + fields: [ 'videoId' ] + } + ], + classMethods: { + associate, + + listWithLimitAndRandom, + + countTotalRequests, + removeAll, + removeByRequestIdsAndPod + } + } + ) + + return RequestVideoEvent +} + +// ------------------------------ STATICS ------------------------------ + +function associate (models) { + this.belongsTo(models.Video, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'CASCADE' + }) +} + +function countTotalRequests (callback) { + const query = {} + return this.count(query).asCallback(callback) +} + +function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) { + const self = this + const Pod = this.sequelize.models.Pod + + // We make a join between videos and authors to find the podId of our video event requests + const podJoins = 'INNER JOIN "Videos" ON "Videos"."authorId" = "Authors"."id" ' + + 'INNER JOIN "RequestVideoEvents" ON "RequestVideoEvents"."videoId" = "Videos"."id"' + + Pod.listRandomPodIdsWithRequest(limitPods, 'Authors', podJoins, function (err, podIds) { + if (err) return callback(err) + + // We don't have friends that have requests + if (podIds.length === 0) return callback(null, []) + + const query = { + order: [ + [ 'id', 'ASC' ] + ], + include: [ + { + model: self.sequelize.models.Video, + include: [ + { + model: self.sequelize.models.Author, + include: [ + { + model: self.sequelize.models.Pod, + where: { + id: { + $in: podIds + } + } + } + ] + } + ] + } + ] + } + + self.findAll(query).asCallback(function (err, requests) { + if (err) return callback(err) + + const requestsGrouped = groupAndTruncateRequests(requests, limitRequestsPerPod) + return callback(err, requestsGrouped) + }) + }) +} + +function removeByRequestIdsAndPod (ids, podId, callback) { + const query = { + where: { + id: { + $in: ids + } + }, + include: [ + { + model: this.sequelize.models.Video, + include: [ + { + model: this.sequelize.models.Author, + where: { + podId + } + } + ] + } + ] + } + + this.destroy(query).asCallback(callback) +} + +function removeAll (callback) { + // Delete all requests + this.truncate({ cascade: true }).asCallback(callback) +} + +// --------------------------------------------------------------------------- + +function groupAndTruncateRequests (events, limitRequestsPerPod) { + const eventsGrouped = {} + + events.forEach(function (event) { + const pod = event.Video.Author.Pod + + if (!eventsGrouped[pod.id]) eventsGrouped[pod.id] = [] + + if (eventsGrouped[pod.id].length < limitRequestsPerPod) { + eventsGrouped[pod.id].push({ + id: event.id, + type: event.type, + count: event.count, + video: event.Video, + pod + }) + } + }) + + return eventsGrouped +} diff --git a/server/models/request-video-qadu.js b/server/models/request-video-qadu.js deleted file mode 100644 index 5d88738aa..000000000 --- a/server/models/request-video-qadu.js +++ /dev/null @@ -1,151 +0,0 @@ -'use strict' - -/* - Request Video for Quick And Dirty Updates like: - - views - - likes - - dislikes - - We can't put it in the same system than basic requests for efficiency. - Moreover we don't want to slow down the basic requests with a lot of views/likes/dislikes requests. - So we put it an independant request scheduler. -*/ - -const values = require('lodash/values') - -const constants = require('../initializers/constants') - -// --------------------------------------------------------------------------- - -module.exports = function (sequelize, DataTypes) { - const RequestVideoQadu = sequelize.define('RequestVideoQadu', - { - type: { - type: DataTypes.ENUM(values(constants.REQUEST_VIDEO_QADU_TYPES)), - allowNull: false - } - }, - { - timestamps: false, - indexes: [ - { - fields: [ 'podId' ] - }, - { - fields: [ 'videoId' ] - } - ], - classMethods: { - associate, - - listWithLimitAndRandom, - - countTotalRequests, - removeAll, - removeByRequestIdsAndPod - } - } - ) - - return RequestVideoQadu -} - -// ------------------------------ STATICS ------------------------------ - -function associate (models) { - this.belongsTo(models.Pod, { - foreignKey: { - name: 'podId', - allowNull: false - }, - onDelete: 'CASCADE' - }) - - this.belongsTo(models.Video, { - foreignKey: { - name: 'videoId', - allowNull: false - }, - onDelete: 'CASCADE' - }) -} - -function countTotalRequests (callback) { - const query = {} - return this.count(query).asCallback(callback) -} - -function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) { - const self = this - const Pod = this.sequelize.models.Pod - - Pod.listRandomPodIdsWithRequest(limitPods, 'RequestVideoQadus', function (err, podIds) { - if (err) return callback(err) - - // We don't have friends that have requests - if (podIds.length === 0) return callback(null, []) - - const query = { - include: [ - { - model: self.sequelize.models.Pod, - where: { - id: { - $in: podIds - } - } - }, - { - model: self.sequelize.models.Video - } - ] - } - - self.findAll(query).asCallback(function (err, requests) { - if (err) return callback(err) - - const requestsGrouped = groupAndTruncateRequests(requests, limitRequestsPerPod) - return callback(err, requestsGrouped) - }) - }) -} - -function removeByRequestIdsAndPod (ids, podId, callback) { - const query = { - where: { - id: { - $in: ids - }, - podId - } - } - - this.destroy(query).asCallback(callback) -} - -function removeAll (callback) { - // Delete all requests - this.truncate({ cascade: true }).asCallback(callback) -} - -// --------------------------------------------------------------------------- - -function groupAndTruncateRequests (requests, limitRequestsPerPod) { - const requestsGrouped = {} - - requests.forEach(function (request) { - const pod = request.Pod - - if (!requestsGrouped[pod.id]) requestsGrouped[pod.id] = [] - - if (requestsGrouped[pod.id].length < limitRequestsPerPod) { - requestsGrouped[pod.id].push({ - request: request, - video: request.Video, - pod - }) - } - }) - - return requestsGrouped -} diff --git a/server/models/request-video-qadu.ts b/server/models/request-video-qadu.ts new file mode 100644 index 000000000..2b1ed07c9 --- /dev/null +++ b/server/models/request-video-qadu.ts @@ -0,0 +1,149 @@ +/* + Request Video for Quick And Dirty Updates like: + - views + - likes + - dislikes + + We can't put it in the same system than basic requests for efficiency. + Moreover we don't want to slow down the basic requests with a lot of views/likes/dislikes requests. + So we put it an independant request scheduler. +*/ + +import { values } from 'lodash' + +import { REQUEST_VIDEO_QADU_TYPES } from '../initializers' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const RequestVideoQadu = sequelize.define('RequestVideoQadu', + { + type: { + type: DataTypes.ENUM(values(REQUEST_VIDEO_QADU_TYPES)), + allowNull: false + } + }, + { + timestamps: false, + indexes: [ + { + fields: [ 'podId' ] + }, + { + fields: [ 'videoId' ] + } + ], + classMethods: { + associate, + + listWithLimitAndRandom, + + countTotalRequests, + removeAll, + removeByRequestIdsAndPod + } + } + ) + + return RequestVideoQadu +} + +// ------------------------------ STATICS ------------------------------ + +function associate (models) { + this.belongsTo(models.Pod, { + foreignKey: { + name: 'podId', + allowNull: false + }, + onDelete: 'CASCADE' + }) + + this.belongsTo(models.Video, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'CASCADE' + }) +} + +function countTotalRequests (callback) { + const query = {} + return this.count(query).asCallback(callback) +} + +function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) { + const self = this + const Pod = this.sequelize.models.Pod + + Pod.listRandomPodIdsWithRequest(limitPods, 'RequestVideoQadus', function (err, podIds) { + if (err) return callback(err) + + // We don't have friends that have requests + if (podIds.length === 0) return callback(null, []) + + const query = { + include: [ + { + model: self.sequelize.models.Pod, + where: { + id: { + $in: podIds + } + } + }, + { + model: self.sequelize.models.Video + } + ] + } + + self.findAll(query).asCallback(function (err, requests) { + if (err) return callback(err) + + const requestsGrouped = groupAndTruncateRequests(requests, limitRequestsPerPod) + return callback(err, requestsGrouped) + }) + }) +} + +function removeByRequestIdsAndPod (ids, podId, callback) { + const query = { + where: { + id: { + $in: ids + }, + podId + } + } + + this.destroy(query).asCallback(callback) +} + +function removeAll (callback) { + // Delete all requests + this.truncate({ cascade: true }).asCallback(callback) +} + +// --------------------------------------------------------------------------- + +function groupAndTruncateRequests (requests, limitRequestsPerPod) { + const requestsGrouped = {} + + requests.forEach(function (request) { + const pod = request.Pod + + if (!requestsGrouped[pod.id]) requestsGrouped[pod.id] = [] + + if (requestsGrouped[pod.id].length < limitRequestsPerPod) { + requestsGrouped[pod.id].push({ + request: request, + video: request.Video, + pod + }) + } + }) + + return requestsGrouped +} diff --git a/server/models/request.js b/server/models/request.js deleted file mode 100644 index 3a047f7ee..000000000 --- a/server/models/request.js +++ /dev/null @@ -1,137 +0,0 @@ -'use strict' - -const values = require('lodash/values') - -const constants = require('../initializers/constants') - -// --------------------------------------------------------------------------- - -module.exports = function (sequelize, DataTypes) { - const Request = sequelize.define('Request', - { - request: { - type: DataTypes.JSON, - allowNull: false - }, - endpoint: { - type: DataTypes.ENUM(values(constants.REQUEST_ENDPOINTS)), - allowNull: false - } - }, - { - classMethods: { - associate, - - listWithLimitAndRandom, - - countTotalRequests, - removeAll, - removeWithEmptyTo - } - } - ) - - return Request -} - -// ------------------------------ STATICS ------------------------------ - -function associate (models) { - this.belongsToMany(models.Pod, { - foreignKey: { - name: 'requestId', - allowNull: false - }, - through: models.RequestToPod, - onDelete: 'CASCADE' - }) -} - -function countTotalRequests (callback) { - // We need to include Pod because there are no cascade delete when a pod is removed - // So we could count requests that do not have existing pod anymore - const query = { - include: [ this.sequelize.models.Pod ] - } - - return this.count(query).asCallback(callback) -} - -function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) { - const self = this - const Pod = this.sequelize.models.Pod - - Pod.listRandomPodIdsWithRequest(limitPods, 'RequestToPods', function (err, podIds) { - if (err) return callback(err) - - // We don't have friends that have requests - if (podIds.length === 0) return callback(null, []) - - // The first x requests of these pods - // It is very important to sort by id ASC to keep the requests order! - const query = { - order: [ - [ 'id', 'ASC' ] - ], - include: [ - { - model: self.sequelize.models.Pod, - where: { - id: { - $in: podIds - } - } - } - ] - } - - self.findAll(query).asCallback(function (err, requests) { - if (err) return callback(err) - - const requestsGrouped = groupAndTruncateRequests(requests, limitRequestsPerPod) - return callback(err, requestsGrouped) - }) - }) -} - -function removeAll (callback) { - // Delete all requests - this.truncate({ cascade: true }).asCallback(callback) -} - -function removeWithEmptyTo (callback) { - if (!callback) callback = function () {} - - const query = { - where: { - id: { - $notIn: [ - this.sequelize.literal('SELECT "requestId" FROM "RequestToPods"') - ] - } - } - } - - this.destroy(query).asCallback(callback) -} - -// --------------------------------------------------------------------------- - -function groupAndTruncateRequests (requests, limitRequestsPerPod) { - const requestsGrouped = {} - - requests.forEach(function (request) { - request.Pods.forEach(function (pod) { - if (!requestsGrouped[pod.id]) requestsGrouped[pod.id] = [] - - if (requestsGrouped[pod.id].length < limitRequestsPerPod) { - requestsGrouped[pod.id].push({ - request, - pod - }) - } - }) - }) - - return requestsGrouped -} diff --git a/server/models/request.ts b/server/models/request.ts new file mode 100644 index 000000000..672f79d11 --- /dev/null +++ b/server/models/request.ts @@ -0,0 +1,135 @@ +import { values } from 'lodash' + +import { REQUEST_ENDPOINTS } from '../initializers' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const Request = sequelize.define('Request', + { + request: { + type: DataTypes.JSON, + allowNull: false + }, + endpoint: { + type: DataTypes.ENUM(values(REQUEST_ENDPOINTS)), + allowNull: false + } + }, + { + classMethods: { + associate, + + listWithLimitAndRandom, + + countTotalRequests, + removeAll, + removeWithEmptyTo + } + } + ) + + return Request +} + +// ------------------------------ STATICS ------------------------------ + +function associate (models) { + this.belongsToMany(models.Pod, { + foreignKey: { + name: 'requestId', + allowNull: false + }, + through: models.RequestToPod, + onDelete: 'CASCADE' + }) +} + +function countTotalRequests (callback) { + // We need to include Pod because there are no cascade delete when a pod is removed + // So we could count requests that do not have existing pod anymore + const query = { + include: [ this.sequelize.models.Pod ] + } + + return this.count(query).asCallback(callback) +} + +function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) { + const self = this + const Pod = this.sequelize.models.Pod + + Pod.listRandomPodIdsWithRequest(limitPods, 'RequestToPods', function (err, podIds) { + if (err) return callback(err) + + // We don't have friends that have requests + if (podIds.length === 0) return callback(null, []) + + // The first x requests of these pods + // It is very important to sort by id ASC to keep the requests order! + const query = { + order: [ + [ 'id', 'ASC' ] + ], + include: [ + { + model: self.sequelize.models.Pod, + where: { + id: { + $in: podIds + } + } + } + ] + } + + self.findAll(query).asCallback(function (err, requests) { + if (err) return callback(err) + + const requestsGrouped = groupAndTruncateRequests(requests, limitRequestsPerPod) + return callback(err, requestsGrouped) + }) + }) +} + +function removeAll (callback) { + // Delete all requests + this.truncate({ cascade: true }).asCallback(callback) +} + +function removeWithEmptyTo (callback) { + if (!callback) callback = function () { /* empty */ } + + const query = { + where: { + id: { + $notIn: [ + this.sequelize.literal('SELECT "requestId" FROM "RequestToPods"') + ] + } + } + } + + this.destroy(query).asCallback(callback) +} + +// --------------------------------------------------------------------------- + +function groupAndTruncateRequests (requests, limitRequestsPerPod) { + const requestsGrouped = {} + + requests.forEach(function (request) { + request.Pods.forEach(function (pod) { + if (!requestsGrouped[pod.id]) requestsGrouped[pod.id] = [] + + if (requestsGrouped[pod.id].length < limitRequestsPerPod) { + requestsGrouped[pod.id].push({ + request, + pod + }) + } + }) + }) + + return requestsGrouped +} diff --git a/server/models/tag.js b/server/models/tag.js deleted file mode 100644 index 145e090c1..000000000 --- a/server/models/tag.js +++ /dev/null @@ -1,76 +0,0 @@ -'use strict' - -const each = require('async/each') - -// --------------------------------------------------------------------------- - -module.exports = function (sequelize, DataTypes) { - const Tag = sequelize.define('Tag', - { - name: { - type: DataTypes.STRING, - allowNull: false - } - }, - { - timestamps: false, - indexes: [ - { - fields: [ 'name' ], - unique: true - } - ], - classMethods: { - associate, - - findOrCreateTags - } - } - ) - - return Tag -} - -// --------------------------------------------------------------------------- - -function associate (models) { - this.belongsToMany(models.Video, { - foreignKey: 'tagId', - through: models.VideoTag, - onDelete: 'cascade' - }) -} - -function findOrCreateTags (tags, transaction, callback) { - if (!callback) { - callback = transaction - transaction = null - } - - const self = this - const tagInstances = [] - - each(tags, function (tag, callbackEach) { - const query = { - where: { - name: tag - }, - defaults: { - name: tag - } - } - - if (transaction) query.transaction = transaction - - self.findOrCreate(query).asCallback(function (err, res) { - if (err) return callbackEach(err) - - // res = [ tag, isCreated ] - const tag = res[0] - tagInstances.push(tag) - return callbackEach() - }) - }, function (err) { - return callback(err, tagInstances) - }) -} diff --git a/server/models/tag.ts b/server/models/tag.ts new file mode 100644 index 000000000..85a0442d2 --- /dev/null +++ b/server/models/tag.ts @@ -0,0 +1,74 @@ +import { each } from 'async' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const Tag = sequelize.define('Tag', + { + name: { + type: DataTypes.STRING, + allowNull: false + } + }, + { + timestamps: false, + indexes: [ + { + fields: [ 'name' ], + unique: true + } + ], + classMethods: { + associate, + + findOrCreateTags + } + } + ) + + return Tag +} + +// --------------------------------------------------------------------------- + +function associate (models) { + this.belongsToMany(models.Video, { + foreignKey: 'tagId', + through: models.VideoTag, + onDelete: 'cascade' + }) +} + +function findOrCreateTags (tags, transaction, callback) { + if (!callback) { + callback = transaction + transaction = null + } + + const self = this + const tagInstances = [] + + each(tags, function (tag, callbackEach) { + const query: any = { + where: { + name: tag + }, + defaults: { + name: tag + } + } + + if (transaction) query.transaction = transaction + + self.findOrCreate(query).asCallback(function (err, res) { + if (err) return callbackEach(err) + + // res = [ tag, isCreated ] + const tag = res[0] + tagInstances.push(tag) + return callbackEach() + }) + }, function (err) { + return callback(err, tagInstances) + }) +} diff --git a/server/models/user-video-rate.js b/server/models/user-video-rate.js deleted file mode 100644 index 84007d70c..000000000 --- a/server/models/user-video-rate.js +++ /dev/null @@ -1,77 +0,0 @@ -'use strict' - -/* - User rates per video. - -*/ - -const values = require('lodash/values') - -const constants = require('../initializers/constants') - -// --------------------------------------------------------------------------- - -module.exports = function (sequelize, DataTypes) { - const UserVideoRate = sequelize.define('UserVideoRate', - { - type: { - type: DataTypes.ENUM(values(constants.VIDEO_RATE_TYPES)), - allowNull: false - } - }, - { - indexes: [ - { - fields: [ 'videoId', 'userId', 'type' ], - unique: true - } - ], - classMethods: { - associate, - - load - } - } - ) - - return UserVideoRate -} - -// ------------------------------ STATICS ------------------------------ - -function associate (models) { - this.belongsTo(models.Video, { - foreignKey: { - name: 'videoId', - allowNull: false - }, - onDelete: 'CASCADE' - }) - - this.belongsTo(models.User, { - foreignKey: { - name: 'userId', - allowNull: false - }, - onDelete: 'CASCADE' - }) -} - -function load (userId, videoId, transaction, callback) { - if (!callback) { - callback = transaction - transaction = null - } - - const query = { - where: { - userId, - videoId - } - } - - const options = {} - if (transaction) options.transaction = transaction - - return this.findOne(query, options).asCallback(callback) -} diff --git a/server/models/user-video-rate.ts b/server/models/user-video-rate.ts new file mode 100644 index 000000000..6603c7862 --- /dev/null +++ b/server/models/user-video-rate.ts @@ -0,0 +1,74 @@ +/* + User rates per video. + +*/ +import { values } from 'lodash' + +import { VIDEO_RATE_TYPES } from '../initializers' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const UserVideoRate = sequelize.define('UserVideoRate', + { + type: { + type: DataTypes.ENUM(values(VIDEO_RATE_TYPES)), + allowNull: false + } + }, + { + indexes: [ + { + fields: [ 'videoId', 'userId', 'type' ], + unique: true + } + ], + classMethods: { + associate, + + load + } + } + ) + + return UserVideoRate +} + +// ------------------------------ STATICS ------------------------------ + +function associate (models) { + this.belongsTo(models.Video, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'CASCADE' + }) + + this.belongsTo(models.User, { + foreignKey: { + name: 'userId', + allowNull: false + }, + onDelete: 'CASCADE' + }) +} + +function load (userId, videoId, transaction, callback) { + if (!callback) { + callback = transaction + transaction = null + } + + const query = { + where: { + userId, + videoId + } + } + + const options: any = {} + if (transaction) options.transaction = transaction + + return this.findOne(query, options).asCallback(callback) +} diff --git a/server/models/user.js b/server/models/user.js deleted file mode 100644 index 8f9c2bf65..000000000 --- a/server/models/user.js +++ /dev/null @@ -1,194 +0,0 @@ -'use strict' - -const values = require('lodash/values') - -const modelUtils = require('./utils') -const constants = require('../initializers/constants') -const peertubeCrypto = require('../helpers/peertube-crypto') -const customUsersValidators = require('../helpers/custom-validators').users - -// --------------------------------------------------------------------------- - -module.exports = function (sequelize, DataTypes) { - const User = sequelize.define('User', - { - password: { - type: DataTypes.STRING, - allowNull: false, - validate: { - passwordValid: function (value) { - const res = customUsersValidators.isUserPasswordValid(value) - if (res === false) throw new Error('Password not valid.') - } - } - }, - username: { - type: DataTypes.STRING, - allowNull: false, - validate: { - usernameValid: function (value) { - const res = customUsersValidators.isUserUsernameValid(value) - if (res === false) throw new Error('Username not valid.') - } - } - }, - email: { - type: DataTypes.STRING(400), - allowNull: false, - validate: { - isEmail: true - } - }, - displayNSFW: { - type: DataTypes.BOOLEAN, - allowNull: false, - defaultValue: false, - validate: { - nsfwValid: function (value) { - const res = customUsersValidators.isUserDisplayNSFWValid(value) - if (res === false) throw new Error('Display NSFW is not valid.') - } - } - }, - role: { - type: DataTypes.ENUM(values(constants.USER_ROLES)), - allowNull: false - } - }, - { - indexes: [ - { - fields: [ 'username' ], - unique: true - }, - { - fields: [ 'email' ], - unique: true - } - ], - classMethods: { - associate, - - countTotal, - getByUsername, - list, - listForApi, - loadById, - loadByUsername, - loadByUsernameOrEmail - }, - instanceMethods: { - isPasswordMatch, - toFormatedJSON, - isAdmin - }, - hooks: { - beforeCreate: beforeCreateOrUpdate, - beforeUpdate: beforeCreateOrUpdate - } - } - ) - - return User -} - -function beforeCreateOrUpdate (user, options, next) { - peertubeCrypto.cryptPassword(user.password, function (err, hash) { - if (err) return next(err) - - user.password = hash - - return next() - }) -} - -// ------------------------------ METHODS ------------------------------ - -function isPasswordMatch (password, callback) { - return peertubeCrypto.comparePassword(password, this.password, callback) -} - -function toFormatedJSON () { - return { - id: this.id, - username: this.username, - email: this.email, - displayNSFW: this.displayNSFW, - role: this.role, - createdAt: this.createdAt - } -} - -function isAdmin () { - return this.role === constants.USER_ROLES.ADMIN -} - -// ------------------------------ STATICS ------------------------------ - -function associate (models) { - this.hasOne(models.Author, { - foreignKey: 'userId', - onDelete: 'cascade' - }) - - this.hasMany(models.OAuthToken, { - foreignKey: 'userId', - onDelete: 'cascade' - }) -} - -function countTotal (callback) { - return this.count().asCallback(callback) -} - -function getByUsername (username) { - const query = { - where: { - username: username - } - } - - return this.findOne(query) -} - -function list (callback) { - return this.find().asCallback(callback) -} - -function listForApi (start, count, sort, callback) { - const query = { - offset: start, - limit: count, - order: [ modelUtils.getSort(sort) ] - } - - return this.findAndCountAll(query).asCallback(function (err, result) { - if (err) return callback(err) - - return callback(null, result.rows, result.count) - }) -} - -function loadById (id, callback) { - return this.findById(id).asCallback(callback) -} - -function loadByUsername (username, callback) { - const query = { - where: { - username: username - } - } - - return this.findOne(query).asCallback(callback) -} - -function loadByUsernameOrEmail (username, email, callback) { - const query = { - where: { - $or: [ { username }, { email } ] - } - } - - return this.findOne(query).asCallback(callback) -} diff --git a/server/models/user.ts b/server/models/user.ts new file mode 100644 index 000000000..d63a50cc4 --- /dev/null +++ b/server/models/user.ts @@ -0,0 +1,197 @@ +import { values } from 'lodash' + +import { getSort } from './utils' +import { USER_ROLES } from '../initializers' +import { + cryptPassword, + comparePassword, + isUserPasswordValid, + isUserUsernameValid, + isUserDisplayNSFWValid +} from '../helpers' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const User = sequelize.define('User', + { + password: { + type: DataTypes.STRING, + allowNull: false, + validate: { + passwordValid: function (value) { + const res = isUserPasswordValid(value) + if (res === false) throw new Error('Password not valid.') + } + } + }, + username: { + type: DataTypes.STRING, + allowNull: false, + validate: { + usernameValid: function (value) { + const res = isUserUsernameValid(value) + if (res === false) throw new Error('Username not valid.') + } + } + }, + email: { + type: DataTypes.STRING(400), + allowNull: false, + validate: { + isEmail: true + } + }, + displayNSFW: { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: false, + validate: { + nsfwValid: function (value) { + const res = isUserDisplayNSFWValid(value) + if (res === false) throw new Error('Display NSFW is not valid.') + } + } + }, + role: { + type: DataTypes.ENUM(values(USER_ROLES)), + allowNull: false + } + }, + { + indexes: [ + { + fields: [ 'username' ], + unique: true + }, + { + fields: [ 'email' ], + unique: true + } + ], + classMethods: { + associate, + + countTotal, + getByUsername, + list, + listForApi, + loadById, + loadByUsername, + loadByUsernameOrEmail + }, + instanceMethods: { + isPasswordMatch, + toFormatedJSON, + isAdmin + }, + hooks: { + beforeCreate: beforeCreateOrUpdate, + beforeUpdate: beforeCreateOrUpdate + } + } + ) + + return User +} + +function beforeCreateOrUpdate (user, options, next) { + cryptPassword(user.password, function (err, hash) { + if (err) return next(err) + + user.password = hash + + return next() + }) +} + +// ------------------------------ METHODS ------------------------------ + +function isPasswordMatch (password, callback) { + return comparePassword(password, this.password, callback) +} + +function toFormatedJSON () { + return { + id: this.id, + username: this.username, + email: this.email, + displayNSFW: this.displayNSFW, + role: this.role, + createdAt: this.createdAt + } +} + +function isAdmin () { + return this.role === USER_ROLES.ADMIN +} + +// ------------------------------ STATICS ------------------------------ + +function associate (models) { + this.hasOne(models.Author, { + foreignKey: 'userId', + onDelete: 'cascade' + }) + + this.hasMany(models.OAuthToken, { + foreignKey: 'userId', + onDelete: 'cascade' + }) +} + +function countTotal (callback) { + return this.count().asCallback(callback) +} + +function getByUsername (username) { + const query = { + where: { + username: username + } + } + + return this.findOne(query) +} + +function list (callback) { + return this.find().asCallback(callback) +} + +function listForApi (start, count, sort, callback) { + const query = { + offset: start, + limit: count, + order: [ getSort(sort) ] + } + + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) +} + +function loadById (id, callback) { + return this.findById(id).asCallback(callback) +} + +function loadByUsername (username, callback) { + const query = { + where: { + username: username + } + } + + return this.findOne(query).asCallback(callback) +} + +function loadByUsernameOrEmail (username, email, callback) { + const query = { + where: { + $or: [ { username }, { email } ] + } + } + + return this.findOne(query).asCallback(callback) +} diff --git a/server/models/utils.js b/server/models/utils.js deleted file mode 100644 index 49636b3d8..000000000 --- a/server/models/utils.js +++ /dev/null @@ -1,25 +0,0 @@ -'use strict' - -const utils = { - getSort -} - -// Translate for example "-name" to [ 'name', 'DESC' ] -function getSort (value) { - let field - let direction - - if (value.substring(0, 1) === '-') { - direction = 'DESC' - field = value.substring(1) - } else { - direction = 'ASC' - field = value - } - - return [ field, direction ] -} - -// --------------------------------------------------------------------------- - -module.exports = utils diff --git a/server/models/utils.ts b/server/models/utils.ts new file mode 100644 index 000000000..601811913 --- /dev/null +++ b/server/models/utils.ts @@ -0,0 +1,21 @@ +// Translate for example "-name" to [ 'name', 'DESC' ] +function getSort (value) { + let field + let direction + + if (value.substring(0, 1) === '-') { + direction = 'DESC' + field = value.substring(1) + } else { + direction = 'ASC' + field = value + } + + return [ field, direction ] +} + +// --------------------------------------------------------------------------- + +export { + getSort +} diff --git a/server/models/video-abuse.js b/server/models/video-abuse.js deleted file mode 100644 index 67cead3af..000000000 --- a/server/models/video-abuse.js +++ /dev/null @@ -1,114 +0,0 @@ -'use strict' - -const constants = require('../initializers/constants') -const modelUtils = require('./utils') -const customVideosValidators = require('../helpers/custom-validators').videos - -module.exports = function (sequelize, DataTypes) { - const VideoAbuse = sequelize.define('VideoAbuse', - { - reporterUsername: { - type: DataTypes.STRING, - allowNull: false, - validate: { - reporterUsernameValid: function (value) { - const res = customVideosValidators.isVideoAbuseReporterUsernameValid(value) - if (res === false) throw new Error('Video abuse reporter username is not valid.') - } - } - }, - reason: { - type: DataTypes.STRING, - allowNull: false, - validate: { - reasonValid: function (value) { - const res = customVideosValidators.isVideoAbuseReasonValid(value) - if (res === false) throw new Error('Video abuse reason is not valid.') - } - } - } - }, - { - indexes: [ - { - fields: [ 'videoId' ] - }, - { - fields: [ 'reporterPodId' ] - } - ], - classMethods: { - associate, - - listForApi - }, - instanceMethods: { - toFormatedJSON - } - } - ) - - return VideoAbuse -} - -// --------------------------------------------------------------------------- - -function associate (models) { - this.belongsTo(models.Pod, { - foreignKey: { - name: 'reporterPodId', - allowNull: true - }, - onDelete: 'cascade' - }) - - this.belongsTo(models.Video, { - foreignKey: { - name: 'videoId', - allowNull: false - }, - onDelete: 'cascade' - }) -} - -function listForApi (start, count, sort, callback) { - const query = { - offset: start, - limit: count, - order: [ modelUtils.getSort(sort) ], - include: [ - { - model: this.sequelize.models.Pod, - required: false - } - ] - } - - return this.findAndCountAll(query).asCallback(function (err, result) { - if (err) return callback(err) - - return callback(null, result.rows, result.count) - }) -} - -function toFormatedJSON () { - let reporterPodHost - - if (this.Pod) { - reporterPodHost = this.Pod.host - } else { - // It means it's our video - reporterPodHost = constants.CONFIG.WEBSERVER.HOST - } - - const json = { - id: this.id, - reporterPodHost, - reason: this.reason, - reporterUsername: this.reporterUsername, - videoId: this.videoId, - createdAt: this.createdAt - } - - return json -} diff --git a/server/models/video-abuse.ts b/server/models/video-abuse.ts new file mode 100644 index 000000000..2a18a293d --- /dev/null +++ b/server/models/video-abuse.ts @@ -0,0 +1,112 @@ +import { CONFIG } from '../initializers' +import { isVideoAbuseReporterUsernameValid, isVideoAbuseReasonValid } from '../helpers' +import { getSort } from './utils' + +module.exports = function (sequelize, DataTypes) { + const VideoAbuse = sequelize.define('VideoAbuse', + { + reporterUsername: { + type: DataTypes.STRING, + allowNull: false, + validate: { + reporterUsernameValid: function (value) { + const res = isVideoAbuseReporterUsernameValid(value) + if (res === false) throw new Error('Video abuse reporter username is not valid.') + } + } + }, + reason: { + type: DataTypes.STRING, + allowNull: false, + validate: { + reasonValid: function (value) { + const res = isVideoAbuseReasonValid(value) + if (res === false) throw new Error('Video abuse reason is not valid.') + } + } + } + }, + { + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'reporterPodId' ] + } + ], + classMethods: { + associate, + + listForApi + }, + instanceMethods: { + toFormatedJSON + } + } + ) + + return VideoAbuse +} + +// --------------------------------------------------------------------------- + +function associate (models) { + this.belongsTo(models.Pod, { + foreignKey: { + name: 'reporterPodId', + allowNull: true + }, + onDelete: 'cascade' + }) + + this.belongsTo(models.Video, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) +} + +function listForApi (start, count, sort, callback) { + const query = { + offset: start, + limit: count, + order: [ getSort(sort) ], + include: [ + { + model: this.sequelize.models.Pod, + required: false + } + ] + } + + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) +} + +function toFormatedJSON () { + let reporterPodHost + + if (this.Pod) { + reporterPodHost = this.Pod.host + } else { + // It means it's our video + reporterPodHost = CONFIG.WEBSERVER.HOST + } + + const json = { + id: this.id, + reporterPodHost, + reason: this.reason, + reporterUsername: this.reporterUsername, + videoId: this.videoId, + createdAt: this.createdAt + } + + return json +} diff --git a/server/models/video-blacklist.js b/server/models/video-blacklist.js deleted file mode 100644 index 02ea15760..000000000 --- a/server/models/video-blacklist.js +++ /dev/null @@ -1,89 +0,0 @@ -'use strict' - -const modelUtils = require('./utils') - -// --------------------------------------------------------------------------- - -module.exports = function (sequelize, DataTypes) { - const BlacklistedVideo = sequelize.define('BlacklistedVideo', - {}, - { - indexes: [ - { - fields: [ 'videoId' ], - unique: true - } - ], - classMethods: { - associate, - - countTotal, - list, - listForApi, - loadById, - loadByVideoId - }, - instanceMethods: { - toFormatedJSON - }, - hooks: {} - } - ) - - return BlacklistedVideo -} - -// ------------------------------ METHODS ------------------------------ - -function toFormatedJSON () { - return { - id: this.id, - videoId: this.videoId, - createdAt: this.createdAt - } -} - -// ------------------------------ STATICS ------------------------------ - -function associate (models) { - this.belongsTo(models.Video, { - foreignKey: 'videoId', - onDelete: 'cascade' - }) -} - -function countTotal (callback) { - return this.count().asCallback(callback) -} - -function list (callback) { - return this.findAll().asCallback(callback) -} - -function listForApi (start, count, sort, callback) { - const query = { - offset: start, - limit: count, - order: [ modelUtils.getSort(sort) ] - } - - return this.findAndCountAll(query).asCallback(function (err, result) { - if (err) return callback(err) - - return callback(null, result.rows, result.count) - }) -} - -function loadById (id, callback) { - return this.findById(id).asCallback(callback) -} - -function loadByVideoId (id, callback) { - const query = { - where: { - videoId: id - } - } - - return this.find(query).asCallback(callback) -} diff --git a/server/models/video-blacklist.ts b/server/models/video-blacklist.ts new file mode 100644 index 000000000..1f00702c7 --- /dev/null +++ b/server/models/video-blacklist.ts @@ -0,0 +1,87 @@ +import { getSort } from './utils' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const BlacklistedVideo = sequelize.define('BlacklistedVideo', + {}, + { + indexes: [ + { + fields: [ 'videoId' ], + unique: true + } + ], + classMethods: { + associate, + + countTotal, + list, + listForApi, + loadById, + loadByVideoId + }, + instanceMethods: { + toFormatedJSON + }, + hooks: {} + } + ) + + return BlacklistedVideo +} + +// ------------------------------ METHODS ------------------------------ + +function toFormatedJSON () { + return { + id: this.id, + videoId: this.videoId, + createdAt: this.createdAt + } +} + +// ------------------------------ STATICS ------------------------------ + +function associate (models) { + this.belongsTo(models.Video, { + foreignKey: 'videoId', + onDelete: 'cascade' + }) +} + +function countTotal (callback) { + return this.count().asCallback(callback) +} + +function list (callback) { + return this.findAll().asCallback(callback) +} + +function listForApi (start, count, sort, callback) { + const query = { + offset: start, + limit: count, + order: [ getSort(sort) ] + } + + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) +} + +function loadById (id, callback) { + return this.findById(id).asCallback(callback) +} + +function loadByVideoId (id, callback) { + const query = { + where: { + videoId: id + } + } + + return this.find(query).asCallback(callback) +} diff --git a/server/models/video-tag.js b/server/models/video-tag.js deleted file mode 100644 index cd9277a6e..000000000 --- a/server/models/video-tag.js +++ /dev/null @@ -1,18 +0,0 @@ -'use strict' - -// --------------------------------------------------------------------------- - -module.exports = function (sequelize, DataTypes) { - const VideoTag = sequelize.define('VideoTag', {}, { - indexes: [ - { - fields: [ 'videoId' ] - }, - { - fields: [ 'tagId' ] - } - ] - }) - - return VideoTag -} diff --git a/server/models/video-tag.ts b/server/models/video-tag.ts new file mode 100644 index 000000000..83ff6053f --- /dev/null +++ b/server/models/video-tag.ts @@ -0,0 +1,14 @@ +module.exports = function (sequelize, DataTypes) { + const VideoTag = sequelize.define('VideoTag', {}, { + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'tagId' ] + } + ] + }) + + return VideoTag +} diff --git a/server/models/video.js b/server/models/video.js deleted file mode 100644 index da4ddb420..000000000 --- a/server/models/video.js +++ /dev/null @@ -1,858 +0,0 @@ -'use strict' - -const Buffer = require('safe-buffer').Buffer -const createTorrent = require('create-torrent') -const ffmpeg = require('fluent-ffmpeg') -const fs = require('fs') -const magnetUtil = require('magnet-uri') -const map = require('lodash/map') -const parallel = require('async/parallel') -const series = require('async/series') -const parseTorrent = require('parse-torrent') -const pathUtils = require('path') -const values = require('lodash/values') - -const constants = require('../initializers/constants') -const logger = require('../helpers/logger') -const friends = require('../lib/friends') -const modelUtils = require('./utils') -const customVideosValidators = require('../helpers/custom-validators').videos -const db = require('../initializers/database') -const jobScheduler = require('../lib/jobs/job-scheduler') - -// --------------------------------------------------------------------------- - -module.exports = function (sequelize, DataTypes) { - const Video = sequelize.define('Video', - { - id: { - type: DataTypes.UUID, - defaultValue: DataTypes.UUIDV4, - primaryKey: true, - validate: { - isUUID: 4 - } - }, - name: { - type: DataTypes.STRING, - allowNull: false, - validate: { - nameValid: function (value) { - const res = customVideosValidators.isVideoNameValid(value) - if (res === false) throw new Error('Video name is not valid.') - } - } - }, - extname: { - type: DataTypes.ENUM(values(constants.CONSTRAINTS_FIELDS.VIDEOS.EXTNAME)), - allowNull: false - }, - remoteId: { - type: DataTypes.UUID, - allowNull: true, - validate: { - isUUID: 4 - } - }, - category: { - type: DataTypes.INTEGER, - allowNull: false, - validate: { - categoryValid: function (value) { - const res = customVideosValidators.isVideoCategoryValid(value) - if (res === false) throw new Error('Video category is not valid.') - } - } - }, - licence: { - type: DataTypes.INTEGER, - allowNull: false, - defaultValue: null, - validate: { - licenceValid: function (value) { - const res = customVideosValidators.isVideoLicenceValid(value) - if (res === false) throw new Error('Video licence is not valid.') - } - } - }, - language: { - type: DataTypes.INTEGER, - allowNull: true, - validate: { - languageValid: function (value) { - const res = customVideosValidators.isVideoLanguageValid(value) - if (res === false) throw new Error('Video language is not valid.') - } - } - }, - nsfw: { - type: DataTypes.BOOLEAN, - allowNull: false, - validate: { - nsfwValid: function (value) { - const res = customVideosValidators.isVideoNSFWValid(value) - if (res === false) throw new Error('Video nsfw attribute is not valid.') - } - } - }, - description: { - type: DataTypes.STRING, - allowNull: false, - validate: { - descriptionValid: function (value) { - const res = customVideosValidators.isVideoDescriptionValid(value) - if (res === false) throw new Error('Video description is not valid.') - } - } - }, - infoHash: { - type: DataTypes.STRING, - allowNull: false, - validate: { - infoHashValid: function (value) { - const res = customVideosValidators.isVideoInfoHashValid(value) - if (res === false) throw new Error('Video info hash is not valid.') - } - } - }, - duration: { - type: DataTypes.INTEGER, - allowNull: false, - validate: { - durationValid: function (value) { - const res = customVideosValidators.isVideoDurationValid(value) - if (res === false) throw new Error('Video duration is not valid.') - } - } - }, - views: { - type: DataTypes.INTEGER, - allowNull: false, - defaultValue: 0, - validate: { - min: 0, - isInt: true - } - }, - likes: { - type: DataTypes.INTEGER, - allowNull: false, - defaultValue: 0, - validate: { - min: 0, - isInt: true - } - }, - dislikes: { - type: DataTypes.INTEGER, - allowNull: false, - defaultValue: 0, - validate: { - min: 0, - isInt: true - } - } - }, - { - indexes: [ - { - fields: [ 'authorId' ] - }, - { - fields: [ 'remoteId' ] - }, - { - fields: [ 'name' ] - }, - { - fields: [ 'createdAt' ] - }, - { - fields: [ 'duration' ] - }, - { - fields: [ 'infoHash' ] - }, - { - fields: [ 'views' ] - }, - { - fields: [ 'likes' ] - } - ], - classMethods: { - associate, - - generateThumbnailFromData, - getDurationFromFile, - list, - listForApi, - listOwnedAndPopulateAuthorAndTags, - listOwnedByAuthor, - load, - loadByHostAndRemoteId, - loadAndPopulateAuthor, - loadAndPopulateAuthorAndPodAndTags, - searchAndPopulateAuthorAndPodAndTags - }, - instanceMethods: { - generateMagnetUri, - getVideoFilename, - getThumbnailName, - getPreviewName, - getTorrentName, - isOwned, - toFormatedJSON, - toAddRemoteJSON, - toUpdateRemoteJSON, - transcodeVideofile, - removeFromBlacklist - }, - hooks: { - beforeValidate, - beforeCreate, - afterDestroy - } - } - ) - - return Video -} - -function beforeValidate (video, options, next) { - // Put a fake infoHash if it does not exists yet - if (video.isOwned() && !video.infoHash) { - // 40 hexa length - video.infoHash = '0123456789abcdef0123456789abcdef01234567' - } - - return next(null) -} - -function beforeCreate (video, options, next) { - const tasks = [] - - if (video.isOwned()) { - const videoPath = pathUtils.join(constants.CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename()) - - tasks.push( - function createVideoTorrent (callback) { - createTorrentFromVideo(video, videoPath, callback) - }, - - function createVideoThumbnail (callback) { - createThumbnail(video, videoPath, callback) - }, - - function createVideoPreview (callback) { - createPreview(video, videoPath, callback) - } - ) - - if (constants.CONFIG.TRANSCODING.ENABLED === true) { - tasks.push( - function createVideoTranscoderJob (callback) { - const dataInput = { - id: video.id - } - - jobScheduler.createJob(options.transaction, 'videoTranscoder', dataInput, callback) - } - ) - } - - return parallel(tasks, next) - } - - return next() -} - -function afterDestroy (video, options, next) { - const tasks = [] - - tasks.push( - function (callback) { - removeThumbnail(video, callback) - } - ) - - if (video.isOwned()) { - tasks.push( - function removeVideoFile (callback) { - removeFile(video, callback) - }, - - function removeVideoTorrent (callback) { - removeTorrent(video, callback) - }, - - function removeVideoPreview (callback) { - removePreview(video, callback) - }, - - function removeVideoToFriends (callback) { - const params = { - remoteId: video.id - } - - friends.removeVideoToFriends(params) - - return callback() - } - ) - } - - parallel(tasks, next) -} - -// ------------------------------ METHODS ------------------------------ - -function associate (models) { - this.belongsTo(models.Author, { - foreignKey: { - name: 'authorId', - allowNull: false - }, - onDelete: 'cascade' - }) - - this.belongsToMany(models.Tag, { - foreignKey: 'videoId', - through: models.VideoTag, - onDelete: 'cascade' - }) - - this.hasMany(models.VideoAbuse, { - foreignKey: { - name: 'videoId', - allowNull: false - }, - onDelete: 'cascade' - }) -} - -function generateMagnetUri () { - let baseUrlHttp, baseUrlWs - - if (this.isOwned()) { - baseUrlHttp = constants.CONFIG.WEBSERVER.URL - baseUrlWs = constants.CONFIG.WEBSERVER.WS + '://' + constants.CONFIG.WEBSERVER.HOSTNAME + ':' + constants.CONFIG.WEBSERVER.PORT - } else { - baseUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + this.Author.Pod.host - baseUrlWs = constants.REMOTE_SCHEME.WS + '://' + this.Author.Pod.host - } - - const xs = baseUrlHttp + constants.STATIC_PATHS.TORRENTS + this.getTorrentName() - const announce = baseUrlWs + '/tracker/socket' - const urlList = [ baseUrlHttp + constants.STATIC_PATHS.WEBSEED + this.getVideoFilename() ] - - const magnetHash = { - xs, - announce, - urlList, - infoHash: this.infoHash, - name: this.name - } - - return magnetUtil.encode(magnetHash) -} - -function getVideoFilename () { - if (this.isOwned()) return this.id + this.extname - - return this.remoteId + this.extname -} - -function getThumbnailName () { - // We always have a copy of the thumbnail - return this.id + '.jpg' -} - -function getPreviewName () { - const extension = '.jpg' - - if (this.isOwned()) return this.id + extension - - return this.remoteId + extension -} - -function getTorrentName () { - const extension = '.torrent' - - if (this.isOwned()) return this.id + extension - - return this.remoteId + extension -} - -function isOwned () { - return this.remoteId === null -} - -function toFormatedJSON () { - let podHost - - if (this.Author.Pod) { - podHost = this.Author.Pod.host - } else { - // It means it's our video - podHost = constants.CONFIG.WEBSERVER.HOST - } - - // Maybe our pod is not up to date and there are new categories since our version - let categoryLabel = constants.VIDEO_CATEGORIES[this.category] - if (!categoryLabel) categoryLabel = 'Misc' - - // Maybe our pod is not up to date and there are new licences since our version - let licenceLabel = constants.VIDEO_LICENCES[this.licence] - if (!licenceLabel) licenceLabel = 'Unknown' - - // Language is an optional attribute - let languageLabel = constants.VIDEO_LANGUAGES[this.language] - if (!languageLabel) languageLabel = 'Unknown' - - const json = { - id: this.id, - name: this.name, - category: this.category, - categoryLabel, - licence: this.licence, - licenceLabel, - language: this.language, - languageLabel, - nsfw: this.nsfw, - description: this.description, - podHost, - isLocal: this.isOwned(), - magnetUri: this.generateMagnetUri(), - author: this.Author.name, - duration: this.duration, - views: this.views, - likes: this.likes, - dislikes: this.dislikes, - tags: map(this.Tags, 'name'), - thumbnailPath: pathUtils.join(constants.STATIC_PATHS.THUMBNAILS, this.getThumbnailName()), - createdAt: this.createdAt, - updatedAt: this.updatedAt - } - - return json -} - -function toAddRemoteJSON (callback) { - const self = this - - // Get thumbnail data to send to the other pod - const thumbnailPath = pathUtils.join(constants.CONFIG.STORAGE.THUMBNAILS_DIR, this.getThumbnailName()) - fs.readFile(thumbnailPath, function (err, thumbnailData) { - if (err) { - logger.error('Cannot read the thumbnail of the video') - return callback(err) - } - - const remoteVideo = { - name: self.name, - category: self.category, - licence: self.licence, - language: self.language, - nsfw: self.nsfw, - description: self.description, - infoHash: self.infoHash, - remoteId: self.id, - author: self.Author.name, - duration: self.duration, - thumbnailData: thumbnailData.toString('binary'), - tags: map(self.Tags, 'name'), - createdAt: self.createdAt, - updatedAt: self.updatedAt, - extname: self.extname, - views: self.views, - likes: self.likes, - dislikes: self.dislikes - } - - return callback(null, remoteVideo) - }) -} - -function toUpdateRemoteJSON (callback) { - const json = { - name: this.name, - category: this.category, - licence: this.licence, - language: this.language, - nsfw: this.nsfw, - description: this.description, - infoHash: this.infoHash, - remoteId: this.id, - author: this.Author.name, - duration: this.duration, - tags: map(this.Tags, 'name'), - createdAt: this.createdAt, - updatedAt: this.updatedAt, - extname: this.extname, - views: this.views, - likes: this.likes, - dislikes: this.dislikes - } - - return json -} - -function transcodeVideofile (finalCallback) { - const video = this - - const videosDirectory = constants.CONFIG.STORAGE.VIDEOS_DIR - const newExtname = '.mp4' - const videoInputPath = pathUtils.join(videosDirectory, video.getVideoFilename()) - const videoOutputPath = pathUtils.join(videosDirectory, video.id + '-transcoded' + newExtname) - - ffmpeg(videoInputPath) - .output(videoOutputPath) - .videoCodec('libx264') - .outputOption('-threads ' + constants.CONFIG.TRANSCODING.THREADS) - .outputOption('-movflags faststart') - .on('error', finalCallback) - .on('end', function () { - series([ - function removeOldFile (callback) { - fs.unlink(videoInputPath, callback) - }, - - function moveNewFile (callback) { - // Important to do this before getVideoFilename() to take in account the new file extension - video.set('extname', newExtname) - - const newVideoPath = pathUtils.join(videosDirectory, video.getVideoFilename()) - fs.rename(videoOutputPath, newVideoPath, callback) - }, - - function torrent (callback) { - const newVideoPath = pathUtils.join(videosDirectory, video.getVideoFilename()) - createTorrentFromVideo(video, newVideoPath, callback) - }, - - function videoExtension (callback) { - video.save().asCallback(callback) - } - - ], function (err) { - if (err) { - // Autodescruction... - video.destroy().asCallback(function (err) { - if (err) logger.error('Cannot destruct video after transcoding failure.', { error: err }) - }) - - return finalCallback(err) - } - - return finalCallback(null) - }) - }) - .run() -} - -// ------------------------------ STATICS ------------------------------ - -function generateThumbnailFromData (video, thumbnailData, callback) { - // Creating the thumbnail for a remote video - - const thumbnailName = video.getThumbnailName() - const thumbnailPath = pathUtils.join(constants.CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) - fs.writeFile(thumbnailPath, Buffer.from(thumbnailData, 'binary'), function (err) { - if (err) return callback(err) - - return callback(null, thumbnailName) - }) -} - -function getDurationFromFile (videoPath, callback) { - ffmpeg.ffprobe(videoPath, function (err, metadata) { - if (err) return callback(err) - - return callback(null, Math.floor(metadata.format.duration)) - }) -} - -function list (callback) { - return this.findAll().asCallback(callback) -} - -function listForApi (start, count, sort, callback) { - // Exclude Blakclisted videos from the list - const query = { - offset: start, - limit: count, - distinct: true, // For the count, a video can have many tags - order: [ modelUtils.getSort(sort), [ this.sequelize.models.Tag, 'name', 'ASC' ] ], - include: [ - { - model: this.sequelize.models.Author, - include: [ { model: this.sequelize.models.Pod, required: false } ] - }, - - this.sequelize.models.Tag - ], - where: createBaseVideosWhere.call(this) - } - - return this.findAndCountAll(query).asCallback(function (err, result) { - if (err) return callback(err) - - return callback(null, result.rows, result.count) - }) -} - -function loadByHostAndRemoteId (fromHost, remoteId, callback) { - const query = { - where: { - remoteId: remoteId - }, - include: [ - { - model: this.sequelize.models.Author, - include: [ - { - model: this.sequelize.models.Pod, - required: true, - where: { - host: fromHost - } - } - ] - } - ] - } - - return this.findOne(query).asCallback(callback) -} - -function listOwnedAndPopulateAuthorAndTags (callback) { - // If remoteId is null this is *our* video - const query = { - where: { - remoteId: null - }, - include: [ this.sequelize.models.Author, this.sequelize.models.Tag ] - } - - return this.findAll(query).asCallback(callback) -} - -function listOwnedByAuthor (author, callback) { - const query = { - where: { - remoteId: null - }, - include: [ - { - model: this.sequelize.models.Author, - where: { - name: author - } - } - ] - } - - return this.findAll(query).asCallback(callback) -} - -function load (id, callback) { - return this.findById(id).asCallback(callback) -} - -function loadAndPopulateAuthor (id, callback) { - const options = { - include: [ this.sequelize.models.Author ] - } - - return this.findById(id, options).asCallback(callback) -} - -function loadAndPopulateAuthorAndPodAndTags (id, callback) { - const options = { - include: [ - { - model: this.sequelize.models.Author, - include: [ { model: this.sequelize.models.Pod, required: false } ] - }, - this.sequelize.models.Tag - ] - } - - return this.findById(id, options).asCallback(callback) -} - -function searchAndPopulateAuthorAndPodAndTags (value, field, start, count, sort, callback) { - const podInclude = { - model: this.sequelize.models.Pod, - required: false - } - - const authorInclude = { - model: this.sequelize.models.Author, - include: [ - podInclude - ] - } - - const tagInclude = { - model: this.sequelize.models.Tag - } - - const query = { - where: createBaseVideosWhere.call(this), - offset: start, - limit: count, - distinct: true, // For the count, a video can have many tags - order: [ modelUtils.getSort(sort), [ this.sequelize.models.Tag, 'name', 'ASC' ] ] - } - - // Make an exact search with the magnet - if (field === 'magnetUri') { - const infoHash = magnetUtil.decode(value).infoHash - query.where.infoHash = infoHash - } else if (field === 'tags') { - const escapedValue = this.sequelize.escape('%' + value + '%') - query.where.id.$in = this.sequelize.literal( - '(SELECT "VideoTags"."videoId" FROM "Tags" INNER JOIN "VideoTags" ON "Tags"."id" = "VideoTags"."tagId" WHERE name LIKE ' + escapedValue + ')' - ) - } else if (field === 'host') { - // FIXME: Include our pod? (not stored in the database) - podInclude.where = { - host: { - $like: '%' + value + '%' - } - } - podInclude.required = true - } else if (field === 'author') { - authorInclude.where = { - name: { - $like: '%' + value + '%' - } - } - - // authorInclude.or = true - } else { - query.where[field] = { - $like: '%' + value + '%' - } - } - - query.include = [ - authorInclude, tagInclude - ] - - if (tagInclude.where) { - // query.include.push([ this.sequelize.models.Tag ]) - } - - return this.findAndCountAll(query).asCallback(function (err, result) { - if (err) return callback(err) - - return callback(null, result.rows, result.count) - }) -} - -// --------------------------------------------------------------------------- - -function createBaseVideosWhere () { - return { - id: { - $notIn: this.sequelize.literal( - '(SELECT "BlacklistedVideos"."videoId" FROM "BlacklistedVideos")' - ) - } - } -} - -function removeThumbnail (video, callback) { - const thumbnailPath = pathUtils.join(constants.CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()) - fs.unlink(thumbnailPath, callback) -} - -function removeFile (video, callback) { - const filePath = pathUtils.join(constants.CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename()) - fs.unlink(filePath, callback) -} - -function removeTorrent (video, callback) { - const torrenPath = pathUtils.join(constants.CONFIG.STORAGE.TORRENTS_DIR, video.getTorrentName()) - fs.unlink(torrenPath, callback) -} - -function removePreview (video, callback) { - // Same name than video thumnail - fs.unlink(constants.CONFIG.STORAGE.PREVIEWS_DIR + video.getPreviewName(), callback) -} - -function createTorrentFromVideo (video, videoPath, callback) { - const options = { - announceList: [ - [ constants.CONFIG.WEBSERVER.WS + '://' + constants.CONFIG.WEBSERVER.HOSTNAME + ':' + constants.CONFIG.WEBSERVER.PORT + '/tracker/socket' ] - ], - urlList: [ - constants.CONFIG.WEBSERVER.URL + constants.STATIC_PATHS.WEBSEED + video.getVideoFilename() - ] - } - - createTorrent(videoPath, options, function (err, torrent) { - if (err) return callback(err) - - const filePath = pathUtils.join(constants.CONFIG.STORAGE.TORRENTS_DIR, video.getTorrentName()) - fs.writeFile(filePath, torrent, function (err) { - if (err) return callback(err) - - const parsedTorrent = parseTorrent(torrent) - video.set('infoHash', parsedTorrent.infoHash) - video.validate().asCallback(callback) - }) - }) -} - -function createPreview (video, videoPath, callback) { - generateImage(video, videoPath, constants.CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName(), callback) -} - -function createThumbnail (video, videoPath, callback) { - generateImage(video, videoPath, constants.CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName(), constants.THUMBNAILS_SIZE, callback) -} - -function generateImage (video, videoPath, folder, imageName, size, callback) { - const options = { - filename: imageName, - count: 1, - folder - } - - if (!callback) { - callback = size - } else { - options.size = size - } - - ffmpeg(videoPath) - .on('error', callback) - .on('end', function () { - callback(null, imageName) - }) - .thumbnail(options) -} - -function removeFromBlacklist (video, callback) { - // Find the blacklisted video - db.BlacklistedVideo.loadByVideoId(video.id, function (err, video) { - // If an error occured, stop here - if (err) { - logger.error('Error when fetching video from blacklist.', { error: err }) - return callback(err) - } - - // If we found the video, remove it from the blacklist - if (video) { - video.destroy().asCallback(callback) - } else { - // If haven't found it, simply ignore it and do nothing - return callback() - } - }) -} diff --git a/server/models/video.ts b/server/models/video.ts new file mode 100644 index 000000000..1e29f1355 --- /dev/null +++ b/server/models/video.ts @@ -0,0 +1,873 @@ +import safeBuffer = require('safe-buffer') +const Buffer = safeBuffer.Buffer +import createTorrent = require('create-torrent') +import ffmpeg = require('fluent-ffmpeg') +import fs = require('fs') +import magnetUtil = require('magnet-uri') +import { map, values } from 'lodash' +import { parallel, series } from 'async' +import parseTorrent = require('parse-torrent') +import { join } from 'path' + +const db = require('../initializers/database') +import { + logger, + isVideoNameValid, + isVideoCategoryValid, + isVideoLicenceValid, + isVideoLanguageValid, + isVideoNSFWValid, + isVideoDescriptionValid, + isVideoInfoHashValid, + isVideoDurationValid +} from '../helpers' +import { + CONSTRAINTS_FIELDS, + CONFIG, + REMOTE_SCHEME, + STATIC_PATHS, + VIDEO_CATEGORIES, + VIDEO_LICENCES, + VIDEO_LANGUAGES, + THUMBNAILS_SIZE +} from '../initializers' +import { JobScheduler, removeVideoToFriends } from '../lib' +import { getSort } from './utils' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const Video = sequelize.define('Video', + { + id: { + type: DataTypes.UUID, + defaultValue: DataTypes.UUIDV4, + primaryKey: true, + validate: { + isUUID: 4 + } + }, + name: { + type: DataTypes.STRING, + allowNull: false, + validate: { + nameValid: function (value) { + const res = isVideoNameValid(value) + if (res === false) throw new Error('Video name is not valid.') + } + } + }, + extname: { + type: DataTypes.ENUM(values(CONSTRAINTS_FIELDS.VIDEOS.EXTNAME)), + allowNull: false + }, + remoteId: { + type: DataTypes.UUID, + allowNull: true, + validate: { + isUUID: 4 + } + }, + category: { + type: DataTypes.INTEGER, + allowNull: false, + validate: { + categoryValid: function (value) { + const res = isVideoCategoryValid(value) + if (res === false) throw new Error('Video category is not valid.') + } + } + }, + licence: { + type: DataTypes.INTEGER, + allowNull: false, + defaultValue: null, + validate: { + licenceValid: function (value) { + const res = isVideoLicenceValid(value) + if (res === false) throw new Error('Video licence is not valid.') + } + } + }, + language: { + type: DataTypes.INTEGER, + allowNull: true, + validate: { + languageValid: function (value) { + const res = isVideoLanguageValid(value) + if (res === false) throw new Error('Video language is not valid.') + } + } + }, + nsfw: { + type: DataTypes.BOOLEAN, + allowNull: false, + validate: { + nsfwValid: function (value) { + const res = isVideoNSFWValid(value) + if (res === false) throw new Error('Video nsfw attribute is not valid.') + } + } + }, + description: { + type: DataTypes.STRING, + allowNull: false, + validate: { + descriptionValid: function (value) { + const res = isVideoDescriptionValid(value) + if (res === false) throw new Error('Video description is not valid.') + } + } + }, + infoHash: { + type: DataTypes.STRING, + allowNull: false, + validate: { + infoHashValid: function (value) { + const res = isVideoInfoHashValid(value) + if (res === false) throw new Error('Video info hash is not valid.') + } + } + }, + duration: { + type: DataTypes.INTEGER, + allowNull: false, + validate: { + durationValid: function (value) { + const res = isVideoDurationValid(value) + if (res === false) throw new Error('Video duration is not valid.') + } + } + }, + views: { + type: DataTypes.INTEGER, + allowNull: false, + defaultValue: 0, + validate: { + min: 0, + isInt: true + } + }, + likes: { + type: DataTypes.INTEGER, + allowNull: false, + defaultValue: 0, + validate: { + min: 0, + isInt: true + } + }, + dislikes: { + type: DataTypes.INTEGER, + allowNull: false, + defaultValue: 0, + validate: { + min: 0, + isInt: true + } + } + }, + { + indexes: [ + { + fields: [ 'authorId' ] + }, + { + fields: [ 'remoteId' ] + }, + { + fields: [ 'name' ] + }, + { + fields: [ 'createdAt' ] + }, + { + fields: [ 'duration' ] + }, + { + fields: [ 'infoHash' ] + }, + { + fields: [ 'views' ] + }, + { + fields: [ 'likes' ] + } + ], + classMethods: { + associate, + + generateThumbnailFromData, + getDurationFromFile, + list, + listForApi, + listOwnedAndPopulateAuthorAndTags, + listOwnedByAuthor, + load, + loadByHostAndRemoteId, + loadAndPopulateAuthor, + loadAndPopulateAuthorAndPodAndTags, + searchAndPopulateAuthorAndPodAndTags + }, + instanceMethods: { + generateMagnetUri, + getVideoFilename, + getThumbnailName, + getPreviewName, + getTorrentName, + isOwned, + toFormatedJSON, + toAddRemoteJSON, + toUpdateRemoteJSON, + transcodeVideofile, + removeFromBlacklist + }, + hooks: { + beforeValidate, + beforeCreate, + afterDestroy + } + } + ) + + return Video +} + +function beforeValidate (video, options, next) { + // Put a fake infoHash if it does not exists yet + if (video.isOwned() && !video.infoHash) { + // 40 hexa length + video.infoHash = '0123456789abcdef0123456789abcdef01234567' + } + + return next(null) +} + +function beforeCreate (video, options, next) { + const tasks = [] + + if (video.isOwned()) { + const videoPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename()) + + tasks.push( + function createVideoTorrent (callback) { + createTorrentFromVideo(video, videoPath, callback) + }, + + function createVideoThumbnail (callback) { + createThumbnail(video, videoPath, callback) + }, + + function createVideoPreview (callback) { + createPreview(video, videoPath, callback) + } + ) + + if (CONFIG.TRANSCODING.ENABLED === true) { + tasks.push( + function createVideoTranscoderJob (callback) { + const dataInput = { + id: video.id + } + + JobScheduler.Instance.createJob(options.transaction, 'videoTranscoder', dataInput, callback) + } + ) + } + + return parallel(tasks, next) + } + + return next() +} + +function afterDestroy (video, options, next) { + const tasks = [] + + tasks.push( + function (callback) { + removeThumbnail(video, callback) + } + ) + + if (video.isOwned()) { + tasks.push( + function removeVideoFile (callback) { + removeFile(video, callback) + }, + + function removeVideoTorrent (callback) { + removeTorrent(video, callback) + }, + + function removeVideoPreview (callback) { + removePreview(video, callback) + }, + + function removeVideoToFriends (callback) { + const params = { + remoteId: video.id + } + + removeVideoToFriends(params) + + return callback() + } + ) + } + + parallel(tasks, next) +} + +// ------------------------------ METHODS ------------------------------ + +function associate (models) { + this.belongsTo(models.Author, { + foreignKey: { + name: 'authorId', + allowNull: false + }, + onDelete: 'cascade' + }) + + this.belongsToMany(models.Tag, { + foreignKey: 'videoId', + through: models.VideoTag, + onDelete: 'cascade' + }) + + this.hasMany(models.VideoAbuse, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) +} + +function generateMagnetUri () { + let baseUrlHttp + let baseUrlWs + + if (this.isOwned()) { + baseUrlHttp = CONFIG.WEBSERVER.URL + baseUrlWs = CONFIG.WEBSERVER.WS + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT + } else { + baseUrlHttp = REMOTE_SCHEME.HTTP + '://' + this.Author.Pod.host + baseUrlWs = REMOTE_SCHEME.WS + '://' + this.Author.Pod.host + } + + const xs = baseUrlHttp + STATIC_PATHS.TORRENTS + this.getTorrentName() + const announce = baseUrlWs + '/tracker/socket' + const urlList = [ baseUrlHttp + STATIC_PATHS.WEBSEED + this.getVideoFilename() ] + + const magnetHash = { + xs, + announce, + urlList, + infoHash: this.infoHash, + name: this.name + } + + return magnetUtil.encode(magnetHash) +} + +function getVideoFilename () { + if (this.isOwned()) return this.id + this.extname + + return this.remoteId + this.extname +} + +function getThumbnailName () { + // We always have a copy of the thumbnail + return this.id + '.jpg' +} + +function getPreviewName () { + const extension = '.jpg' + + if (this.isOwned()) return this.id + extension + + return this.remoteId + extension +} + +function getTorrentName () { + const extension = '.torrent' + + if (this.isOwned()) return this.id + extension + + return this.remoteId + extension +} + +function isOwned () { + return this.remoteId === null +} + +function toFormatedJSON () { + let podHost + + if (this.Author.Pod) { + podHost = this.Author.Pod.host + } else { + // It means it's our video + podHost = CONFIG.WEBSERVER.HOST + } + + // Maybe our pod is not up to date and there are new categories since our version + let categoryLabel = VIDEO_CATEGORIES[this.category] + if (!categoryLabel) categoryLabel = 'Misc' + + // Maybe our pod is not up to date and there are new licences since our version + let licenceLabel = VIDEO_LICENCES[this.licence] + if (!licenceLabel) licenceLabel = 'Unknown' + + // Language is an optional attribute + let languageLabel = VIDEO_LANGUAGES[this.language] + if (!languageLabel) languageLabel = 'Unknown' + + const json = { + id: this.id, + name: this.name, + category: this.category, + categoryLabel, + licence: this.licence, + licenceLabel, + language: this.language, + languageLabel, + nsfw: this.nsfw, + description: this.description, + podHost, + isLocal: this.isOwned(), + magnetUri: this.generateMagnetUri(), + author: this.Author.name, + duration: this.duration, + views: this.views, + likes: this.likes, + dislikes: this.dislikes, + tags: map(this.Tags, 'name'), + thumbnailPath: join(STATIC_PATHS.THUMBNAILS, this.getThumbnailName()), + createdAt: this.createdAt, + updatedAt: this.updatedAt + } + + return json +} + +function toAddRemoteJSON (callback) { + const self = this + + // Get thumbnail data to send to the other pod + const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, this.getThumbnailName()) + fs.readFile(thumbnailPath, function (err, thumbnailData) { + if (err) { + logger.error('Cannot read the thumbnail of the video') + return callback(err) + } + + const remoteVideo = { + name: self.name, + category: self.category, + licence: self.licence, + language: self.language, + nsfw: self.nsfw, + description: self.description, + infoHash: self.infoHash, + remoteId: self.id, + author: self.Author.name, + duration: self.duration, + thumbnailData: thumbnailData.toString('binary'), + tags: map(self.Tags, 'name'), + createdAt: self.createdAt, + updatedAt: self.updatedAt, + extname: self.extname, + views: self.views, + likes: self.likes, + dislikes: self.dislikes + } + + return callback(null, remoteVideo) + }) +} + +function toUpdateRemoteJSON (callback) { + const json = { + name: this.name, + category: this.category, + licence: this.licence, + language: this.language, + nsfw: this.nsfw, + description: this.description, + infoHash: this.infoHash, + remoteId: this.id, + author: this.Author.name, + duration: this.duration, + tags: map(this.Tags, 'name'), + createdAt: this.createdAt, + updatedAt: this.updatedAt, + extname: this.extname, + views: this.views, + likes: this.likes, + dislikes: this.dislikes + } + + return json +} + +function transcodeVideofile (finalCallback) { + const video = this + + const videosDirectory = CONFIG.STORAGE.VIDEOS_DIR + const newExtname = '.mp4' + const videoInputPath = join(videosDirectory, video.getVideoFilename()) + const videoOutputPath = join(videosDirectory, video.id + '-transcoded' + newExtname) + + ffmpeg(videoInputPath) + .output(videoOutputPath) + .videoCodec('libx264') + .outputOption('-threads ' + CONFIG.TRANSCODING.THREADS) + .outputOption('-movflags faststart') + .on('error', finalCallback) + .on('end', function () { + series([ + function removeOldFile (callback) { + fs.unlink(videoInputPath, callback) + }, + + function moveNewFile (callback) { + // Important to do this before getVideoFilename() to take in account the new file extension + video.set('extname', newExtname) + + const newVideoPath = join(videosDirectory, video.getVideoFilename()) + fs.rename(videoOutputPath, newVideoPath, callback) + }, + + function torrent (callback) { + const newVideoPath = join(videosDirectory, video.getVideoFilename()) + createTorrentFromVideo(video, newVideoPath, callback) + }, + + function videoExtension (callback) { + video.save().asCallback(callback) + } + + ], function (err) { + if (err) { + // Autodescruction... + video.destroy().asCallback(function (err) { + if (err) logger.error('Cannot destruct video after transcoding failure.', { error: err }) + }) + + return finalCallback(err) + } + + return finalCallback(null) + }) + }) + .run() +} + +// ------------------------------ STATICS ------------------------------ + +function generateThumbnailFromData (video, thumbnailData, callback) { + // Creating the thumbnail for a remote video + + const thumbnailName = video.getThumbnailName() + const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) + fs.writeFile(thumbnailPath, Buffer.from(thumbnailData, 'binary'), function (err) { + if (err) return callback(err) + + return callback(null, thumbnailName) + }) +} + +function getDurationFromFile (videoPath, callback) { + ffmpeg.ffprobe(videoPath, function (err, metadata) { + if (err) return callback(err) + + return callback(null, Math.floor(metadata.format.duration)) + }) +} + +function list (callback) { + return this.findAll().asCallback(callback) +} + +function listForApi (start, count, sort, callback) { + // Exclude Blakclisted videos from the list + const query = { + offset: start, + limit: count, + distinct: true, // For the count, a video can have many tags + order: [ getSort(sort), [ this.sequelize.models.Tag, 'name', 'ASC' ] ], + include: [ + { + model: this.sequelize.models.Author, + include: [ { model: this.sequelize.models.Pod, required: false } ] + }, + + this.sequelize.models.Tag + ], + where: createBaseVideosWhere.call(this) + } + + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) +} + +function loadByHostAndRemoteId (fromHost, remoteId, callback) { + const query = { + where: { + remoteId: remoteId + }, + include: [ + { + model: this.sequelize.models.Author, + include: [ + { + model: this.sequelize.models.Pod, + required: true, + where: { + host: fromHost + } + } + ] + } + ] + } + + return this.findOne(query).asCallback(callback) +} + +function listOwnedAndPopulateAuthorAndTags (callback) { + // If remoteId is null this is *our* video + const query = { + where: { + remoteId: null + }, + include: [ this.sequelize.models.Author, this.sequelize.models.Tag ] + } + + return this.findAll(query).asCallback(callback) +} + +function listOwnedByAuthor (author, callback) { + const query = { + where: { + remoteId: null + }, + include: [ + { + model: this.sequelize.models.Author, + where: { + name: author + } + } + ] + } + + return this.findAll(query).asCallback(callback) +} + +function load (id, callback) { + return this.findById(id).asCallback(callback) +} + +function loadAndPopulateAuthor (id, callback) { + const options = { + include: [ this.sequelize.models.Author ] + } + + return this.findById(id, options).asCallback(callback) +} + +function loadAndPopulateAuthorAndPodAndTags (id, callback) { + const options = { + include: [ + { + model: this.sequelize.models.Author, + include: [ { model: this.sequelize.models.Pod, required: false } ] + }, + this.sequelize.models.Tag + ] + } + + return this.findById(id, options).asCallback(callback) +} + +function searchAndPopulateAuthorAndPodAndTags (value, field, start, count, sort, callback) { + const podInclude: any = { + model: this.sequelize.models.Pod, + required: false + } + + const authorInclude: any = { + model: this.sequelize.models.Author, + include: [ + podInclude + ] + } + + const tagInclude: any = { + model: this.sequelize.models.Tag + } + + const query: any = { + where: createBaseVideosWhere.call(this), + offset: start, + limit: count, + distinct: true, // For the count, a video can have many tags + order: [ getSort(sort), [ this.sequelize.models.Tag, 'name', 'ASC' ] ] + } + + // Make an exact search with the magnet + if (field === 'magnetUri') { + const infoHash = magnetUtil.decode(value).infoHash + query.where.infoHash = infoHash + } else if (field === 'tags') { + const escapedValue = this.sequelize.escape('%' + value + '%') + query.where.id.$in = this.sequelize.literal( + '(SELECT "VideoTags"."videoId" FROM "Tags" INNER JOIN "VideoTags" ON "Tags"."id" = "VideoTags"."tagId" WHERE name LIKE ' + escapedValue + ')' + ) + } else if (field === 'host') { + // FIXME: Include our pod? (not stored in the database) + podInclude.where = { + host: { + $like: '%' + value + '%' + } + } + podInclude.required = true + } else if (field === 'author') { + authorInclude.where = { + name: { + $like: '%' + value + '%' + } + } + + // authorInclude.or = true + } else { + query.where[field] = { + $like: '%' + value + '%' + } + } + + query.include = [ + authorInclude, tagInclude + ] + + if (tagInclude.where) { + // query.include.push([ this.sequelize.models.Tag ]) + } + + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) +} + +// --------------------------------------------------------------------------- + +function createBaseVideosWhere () { + return { + id: { + $notIn: this.sequelize.literal( + '(SELECT "BlacklistedVideos"."videoId" FROM "BlacklistedVideos")' + ) + } + } +} + +function removeThumbnail (video, callback) { + const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()) + fs.unlink(thumbnailPath, callback) +} + +function removeFile (video, callback) { + const filePath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename()) + fs.unlink(filePath, callback) +} + +function removeTorrent (video, callback) { + const torrenPath = join(CONFIG.STORAGE.TORRENTS_DIR, video.getTorrentName()) + fs.unlink(torrenPath, callback) +} + +function removePreview (video, callback) { + // Same name than video thumnail + fs.unlink(CONFIG.STORAGE.PREVIEWS_DIR + video.getPreviewName(), callback) +} + +function createTorrentFromVideo (video, videoPath, callback) { + const options = { + announceList: [ + [ CONFIG.WEBSERVER.WS + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT + '/tracker/socket' ] + ], + urlList: [ + CONFIG.WEBSERVER.URL + STATIC_PATHS.WEBSEED + video.getVideoFilename() + ] + } + + createTorrent(videoPath, options, function (err, torrent) { + if (err) return callback(err) + + const filePath = join(CONFIG.STORAGE.TORRENTS_DIR, video.getTorrentName()) + fs.writeFile(filePath, torrent, function (err) { + if (err) return callback(err) + + const parsedTorrent = parseTorrent(torrent) + video.set('infoHash', parsedTorrent.infoHash) + video.validate().asCallback(callback) + }) + }) +} + +function createPreview (video, videoPath, callback) { + generateImage(video, videoPath, CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName(), callback) +} + +function createThumbnail (video, videoPath, callback) { + generateImage(video, videoPath, CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName(), THUMBNAILS_SIZE, callback) +} + +function generateImage (video, videoPath, folder, imageName, size, callback?) { + const options: any = { + filename: imageName, + count: 1, + folder + } + + if (!callback) { + callback = size + } else { + options.size = size + } + + ffmpeg(videoPath) + .on('error', callback) + .on('end', function () { + callback(null, imageName) + }) + .thumbnail(options) +} + +function removeFromBlacklist (video, callback) { + // Find the blacklisted video + db.BlacklistedVideo.loadByVideoId(video.id, function (err, video) { + // If an error occured, stop here + if (err) { + logger.error('Error when fetching video from blacklist.', { error: err }) + return callback(err) + } + + // If we found the video, remove it from the blacklist + if (video) { + video.destroy().asCallback(callback) + } else { + // If haven't found it, simply ignore it and do nothing + return callback() + } + }) +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 000000000..039881230 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "module": "commonjs", + "target": "es5", + "noImplicitAny": false, + "sourceMap": false, + "outDir": "./dist", + "lib": [ + "es2015" + ], + "types": [ + "node" + ] + }, + "exclude": [ + "node_modules", + "client" + ] +} diff --git a/tslint.json b/tslint.json new file mode 100644 index 000000000..888779856 --- /dev/null +++ b/tslint.json @@ -0,0 +1,3 @@ +{ + "extends": "tslint-config-standard" +} diff --git a/yarn.lock b/yarn.lock index 284cdb444..c0ce443b3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,34 +2,88 @@ # yarn lockfile v1 -"@types/bluebird@~3.0.36": +"@types/async@^2.0.40": + version "2.0.40" + resolved "https://registry.yarnpkg.com/@types/async/-/async-2.0.40.tgz#ac02de68e66c004a61b7cb16df8b1db3a254cca9" + +"@types/bcrypt@^1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@types/bcrypt/-/bcrypt-1.0.0.tgz#2c523da191db7d41c06d17de235335c985effe9b" + +"@types/bluebird@*", "@types/bluebird@~3.0.36": version "3.0.37" resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.0.37.tgz#2e76b394aa9bea40d04241a31c0887a260283388" +"@types/body-parser@^1.16.3": + version "1.16.3" + resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.16.3.tgz#bc2b9a181f2fa85c80f1ecacd8a05cf1414b85a3" + dependencies: + "@types/express" "*" + "@types/node" "*" + +"@types/config@^0.0.32": + version "0.0.32" + resolved "https://registry.yarnpkg.com/@types/config/-/config-0.0.32.tgz#c106055802d78e234e28374adc4dad460d098558" + "@types/express-serve-static-core@*": version "4.0.44" resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.0.44.tgz#a1c3bd5d80e93c72fba91a03f5412c47f21d4ae7" dependencies: "@types/node" "*" -"@types/express@~4.0.34": +"@types/express@*", "@types/express@^4.0.35", "@types/express@~4.0.34": version "4.0.35" resolved "https://registry.yarnpkg.com/@types/express/-/express-4.0.35.tgz#6267c7b60a51fac473467b3c4a02cd1e441805fe" dependencies: "@types/express-serve-static-core" "*" "@types/serve-static" "*" +"@types/form-data@*": + version "0.0.33" + resolved "https://registry.yarnpkg.com/@types/form-data/-/form-data-0.0.33.tgz#c9ac85b2a5fd18435b8c85d9ecb50e6d6c893ff8" + dependencies: + "@types/node" "*" + "@types/geojson@^1.0.0": version "1.0.2" resolved "https://registry.yarnpkg.com/@types/geojson/-/geojson-1.0.2.tgz#b02d10ab028e2928ac592a051aaa4981a1941d03" +"@types/lodash@*", "@types/lodash@^4.14.64": + version "4.14.64" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.64.tgz#979cf3a3d4a368670840bf9b3e448dc33ffe84ee" + "@types/mime@*": version "0.0.29" resolved "https://registry.yarnpkg.com/@types/mime/-/mime-0.0.29.tgz#fbcfd330573b912ef59eeee14602bface630754b" -"@types/node@*": - version "7.0.14" - resolved "https://registry.yarnpkg.com/@types/node/-/node-7.0.14.tgz#1470fa002a113316ac9d9ad163fc738c7a0de2a4" +"@types/mkdirp@^0.3.29": + version "0.3.29" + resolved "https://registry.yarnpkg.com/@types/mkdirp/-/mkdirp-0.3.29.tgz#7f2ad7ec55f914482fc9b1ec4bb1ae6028d46066" + +"@types/morgan@^1.7.32": + version "1.7.32" + resolved "https://registry.yarnpkg.com/@types/morgan/-/morgan-1.7.32.tgz#fab1ece4dae172e1a377d563d33e3634fa04927d" + dependencies: + "@types/express" "*" + +"@types/node@*", "@types/node@^7.0.18": + version "7.0.18" + resolved "https://registry.yarnpkg.com/@types/node/-/node-7.0.18.tgz#cd67f27d3dc0cfb746f0bdd5e086c4c5d55be173" + +"@types/request@^0.0.43": + version "0.0.43" + resolved "https://registry.yarnpkg.com/@types/request/-/request-0.0.43.tgz#fcc59cfd88e63034e813c6884a0aade2d0f7e935" + dependencies: + "@types/form-data" "*" + "@types/node" "*" + +"@types/sequelize@3": + version "3.4.48" + resolved "https://registry.yarnpkg.com/@types/sequelize/-/sequelize-3.4.48.tgz#f88fac7cc4717d2e87f20f69ebb64aa869e7e4d1" + dependencies: + "@types/bluebird" "*" + "@types/lodash" "*" + "@types/validator" "*" "@types/serve-static@*": version "1.7.31" @@ -38,6 +92,22 @@ "@types/express-serve-static-core" "*" "@types/mime" "*" +"@types/validator@*": + version "6.2.0" + resolved "https://registry.yarnpkg.com/@types/validator/-/validator-6.2.0.tgz#020322fe1929f69889eb675a1bdb5a98394b71f0" + +"@types/winston@^2.3.2": + version "2.3.2" + resolved "https://registry.yarnpkg.com/@types/winston/-/winston-2.3.2.tgz#c162547cb47c0b8a450e681bb9fa7041cd80edfa" + dependencies: + "@types/node" "*" + +"@types/ws@^0.0.41": + version "0.0.41" + resolved "https://registry.yarnpkg.com/@types/ws/-/ws-0.0.41.tgz#88a7e0cd1605bd6ea773110954671394c690db1a" + dependencies: + "@types/node" "*" + abbrev@1: version "1.1.0" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.0.tgz#d0554c2256636e2f56e7c2e5ad183f859428d81f" @@ -198,7 +268,7 @@ aws4@^1.2.1: version "1.6.0" resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e" -babel-code-frame@^6.16.0: +babel-code-frame@^6.16.0, babel-code-frame@^6.22.0: version "6.22.0" resolved "https://registry.yarnpkg.com/babel-code-frame/-/babel-code-frame-6.22.0.tgz#027620bee567a88c32561574e7fd0801d33118e4" dependencies: @@ -523,6 +593,10 @@ colors@1.0.x: version "1.0.3" resolved "https://registry.yarnpkg.com/colors/-/colors-1.0.3.tgz#0433f44d809680fdeb60ed260f1b0c262e82a40b" +colors@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/colors/-/colors-1.1.2.tgz#168a4701756b6a7f51a12ce0c97bfa28c084ed63" + combined-stream@^1.0.5, combined-stream@~1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.5.tgz#938370a57b4a51dea2c77c15d5c5fdf895164009" @@ -785,7 +859,7 @@ dicer@0.2.5: readable-stream "1.1.x" streamsearch "0.1.2" -diff@3.2.0: +diff@3.2.0, diff@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/diff/-/diff-3.2.0.tgz#c9ce393a4b7cbd0b058a725c93df299027868ff9" @@ -796,6 +870,13 @@ doctrine@1.5.0, doctrine@^1.2.2: esutils "^2.0.2" isarray "^1.0.0" +doctrine@^0.7.2: + version "0.7.2" + resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-0.7.2.tgz#7cb860359ba3be90e040b26b729ce4bfa654c523" + dependencies: + esutils "^1.1.6" + isarray "0.0.1" + doctrine@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-2.0.0.tgz#c73d8d2909d22291e1a007a395804da8b665fe63" @@ -1057,6 +1138,10 @@ estraverse@~4.1.0: version "4.1.1" resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-4.1.1.tgz#f6caca728933a850ef90661d0e17982ba47111a2" +esutils@^1.1.6: + version "1.1.6" + resolved "https://registry.yarnpkg.com/esutils/-/esutils-1.1.6.tgz#c01ccaa9ae4b897c6d0c3e210ae52f3c7a844375" + esutils@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.2.tgz#0abf4f1caa5bcb1f7a9d8acc6dea4faaa04bac9b" @@ -1200,6 +1285,12 @@ find-up@^2.0.0: dependencies: locate-path "^2.0.0" +findup-sync@~0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/findup-sync/-/findup-sync-0.3.0.tgz#37930aa5d816b777c03445e1966cc6790a4c0b16" + dependencies: + glob "~5.0.0" + flat-cache@^1.2.1: version "1.2.2" resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-1.2.2.tgz#fa86714e72c21db88601761ecf2f555d1abc6b96" @@ -1333,7 +1424,7 @@ github-from-package@0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce" -glob@7.1.1, glob@^7.0.0, glob@^7.0.3, glob@^7.0.5: +glob@7.1.1, glob@^7.0.0, glob@^7.0.3, glob@^7.0.5, glob@^7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" dependencies: @@ -1344,6 +1435,16 @@ glob@7.1.1, glob@^7.0.0, glob@^7.0.3, glob@^7.0.5: once "^1.3.0" path-is-absolute "^1.0.0" +glob@~5.0.0: + version "5.0.15" + resolved "https://registry.yarnpkg.com/glob/-/glob-5.0.15.tgz#1bc936b9e02f4a603fcc222ecf7633d30b8b93b1" + dependencies: + inflight "^1.0.4" + inherits "2" + minimatch "2 || 3" + once "^1.3.0" + path-is-absolute "^1.0.0" + globals@^9.14.0: version "9.17.0" resolved "https://registry.yarnpkg.com/globals/-/globals-9.17.0.tgz#0c0ca696d9b9bb694d2e5470bd37777caad50286" @@ -1862,7 +1963,7 @@ mime@1.3.4, mime@^1.3.4: version "1.3.4" resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.4.tgz#115f9e3b6b3daf2959983cb38f149a2d40eb5d53" -minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.3: +"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.3.tgz#2a4e4090b96b2db06a9d7df01055a62a77c9b774" dependencies: @@ -2101,7 +2202,7 @@ openssl-wrapper@^0.3.4: version "0.3.4" resolved "https://registry.yarnpkg.com/openssl-wrapper/-/openssl-wrapper-0.3.4.tgz#c01ec98e4dcd2b5dfe0b693f31827200e3b81b07" -optimist@0.6.1: +optimist@0.6.1, optimist@~0.6.0: version "0.6.1" resolved "https://registry.yarnpkg.com/optimist/-/optimist-0.6.1.tgz#da3ea74686fa21a19a111c326e90eb15a0196686" dependencies: @@ -2533,7 +2634,7 @@ resolve-from@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-1.0.1.tgz#26cbfe935d1aeeeabb29bc3fe5aeb01e93d44226" -resolve@^1.1.6, resolve@^1.1.7: +resolve@^1.1.6, resolve@^1.1.7, resolve@^1.3.2: version "1.3.3" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.3.3.tgz#655907c3469a8680dc2de3a275a8fdd69691f0e5" dependencies: @@ -2616,7 +2717,7 @@ semver@4.3.2: version "4.3.2" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7" -semver@5.3.0, semver@^5.0.1, semver@~5.3.0: +semver@5.3.0, semver@^5.0.1, semver@^5.3.0, semver@~5.3.0: version "5.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" @@ -3080,6 +3181,43 @@ tryit@^1.0.1: version "1.0.3" resolved "https://registry.yarnpkg.com/tryit/-/tryit-1.0.3.tgz#393be730a9446fd1ead6da59a014308f36c289cb" +tslib@^1.0.0, tslib@^1.6.0: + version "1.7.0" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.7.0.tgz#6e8366695f72961252b35167b0dd4fbeeafba491" + +tslint-config-standard@^5.0.2: + version "5.0.2" + resolved "https://registry.yarnpkg.com/tslint-config-standard/-/tslint-config-standard-5.0.2.tgz#e98fd5c412a6b973798366dc2c85508cf0ed740f" + dependencies: + tslint-eslint-rules "^4.0.0" + +tslint-eslint-rules@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/tslint-eslint-rules/-/tslint-eslint-rules-4.0.0.tgz#4e0e59ecd5701c9a48c66ed47bdcafb1c635d27b" + dependencies: + doctrine "^0.7.2" + tslib "^1.0.0" + tsutils "^1.4.0" + +tslint@^5.2.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/tslint/-/tslint-5.2.0.tgz#16a2addf20cb748385f544e9a0edab086bc34114" + dependencies: + babel-code-frame "^6.22.0" + colors "^1.1.2" + diff "^3.2.0" + findup-sync "~0.3.0" + glob "^7.1.1" + optimist "~0.6.0" + resolve "^1.3.2" + semver "^5.3.0" + tslib "^1.6.0" + tsutils "^1.8.0" + +tsutils@^1.4.0, tsutils@^1.8.0: + version "1.8.0" + resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-1.8.0.tgz#bf8118ed8e80cd5c9fc7d75728c7963d44ed2f52" + tunnel-agent@^0.4.3: version "0.4.3" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.4.3.tgz#6373db76909fe570e08d73583365ed828a74eeeb" @@ -3125,6 +3263,10 @@ typedarray@^0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" +typescript@~2.2.0: + version "2.2.2" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-2.2.2.tgz#606022508479b55ffa368b58fee963a03dfd7b0c" + uid-number@~0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/uid-number/-/uid-number-0.0.6.tgz#0ea10e8035e8eb5b8e4449f06da1c730663baa81" -- cgit v1.2.3