3 files changed, 82 insertions, 10 deletions

diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts
index 52a1aeb50..608123607 100644
--- a/server/initializers/checker.ts
+++ b/server/initializers/checker.ts
@@ -43,7 +43,7 @@ function checkMissedConfig () {
   const required = [ 'listen.port', 'listen.hostname',
     'webserver.https', 'webserver.hostname', 'webserver.port',
     'trust_proxy',
-    'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password',
+    'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max',
     'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address',
     'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache',
     'log.level',
@@ -51,6 +51,7 @@ function checkMissedConfig () {
     'cache.previews.size', 'admin.email',
     'signup.enabled', 'signup.limit', 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist',
     'transcoding.enabled', 'transcoding.threads',
+    'import.videos.http.enabled',
     'instance.name', 'instance.short_description', 'instance.description', 'instance.terms', 'instance.default_client_route',
     'instance.default_nsfw_policy', 'instance.robots',
     'services.twitter.username', 'services.twitter.whitelisted'
@@ -84,11 +85,11 @@ function checkMissedConfig () {
 async function checkFFmpeg (CONFIG: { TRANSCODING: { ENABLED: boolean } }) {
   const Ffmpeg = require('fluent-ffmpeg')
   const getAvailableCodecsPromise = promisify0(Ffmpeg.getAvailableCodecs)
-
   const codecs = await getAvailableCodecsPromise()
+  const canEncode = [ 'libx264' ]
+
   if (CONFIG.TRANSCODING.ENABLED === false) return undefined
 
-  const canEncode = [ 'libx264' ]
   for (const codec of canEncode) {
     if (codecs[codec] === undefined) {
       throw new Error('Unknown codec ' + codec + ' in FFmpeg.')
@@ -98,6 +99,29 @@ async function checkFFmpeg (CONFIG: { TRANSCODING: { ENABLED: boolean } }) {
       throw new Error('Unavailable encode codec ' + codec + ' in FFmpeg')
     }
   }
+
+  checkFFmpegEncoders()
+}
+
+// Optional encoders, if present, can be used to improve transcoding
+// Here we ask ffmpeg if it detects their presence on the system, so that we can later use them
+let supportedOptionalEncoders: Map<string, boolean>
+async function checkFFmpegEncoders (): Promise<Map<string, boolean>> {
+  if (supportedOptionalEncoders !== undefined) {
+    return supportedOptionalEncoders
+  }
+
+  const Ffmpeg = require('fluent-ffmpeg')
+  const getAvailableEncodersPromise = promisify0(Ffmpeg.getAvailableEncoders)
+  const encoders = await getAvailableEncodersPromise()
+  const optionalEncoders = [ 'libfdk_aac' ]
+  supportedOptionalEncoders = new Map<string, boolean>()
+
+  for (const encoder of optionalEncoders) {
+    supportedOptionalEncoders.set(encoder,
+      encoders[encoder] !== undefined
+    )
+  }
 }
 
 // We get db by param to not import it in this file (import orders)
@@ -126,6 +150,7 @@ async function applicationExist () {
 export {
   checkConfig,
   checkFFmpeg,
+  checkFFmpegEncoders,
   checkMissedConfig,
   clientsExist,
   usersExist,
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 3aa979668..069d9b2e8 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -8,6 +8,7 @@ import { VideoPrivacy } from '../../shared/models/videos'
 import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils'
 import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
 import { invert } from 'lodash'
+import { VideoImportState } from '../../shared/models/videos/video-import-state.enum'
 
 // Use a variable to reload the configuration if we need
 let config: IConfig = require('config')
@@ -36,6 +37,7 @@ const SORTABLE_COLUMNS = {
   VIDEO_ABUSES: [ 'id', 'createdAt' ],
   VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ],
   VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'views', 'likes' ],
+  VIDEO_IMPORTS: [ 'createdAt' ],
   VIDEO_COMMENT_THREADS: [ 'createdAt' ],
   BLACKLISTS: [ 'id', 'name', 'duration', 'views', 'likes', 'dislikes', 'uuid', 'createdAt' ],
   FOLLOWERS: [ 'createdAt' ],
@@ -85,6 +87,7 @@ const JOB_ATTEMPTS: { [ id in JobType ]: number } = {
   'activitypub-follow': 5,
   'video-file-import': 1,
   'video-file': 1,
+  'video-import': 1,
   'email': 5
 }
 const JOB_CONCURRENCY: { [ id in JobType ]: number } = {
@@ -94,18 +97,29 @@ const JOB_CONCURRENCY: { [ id in JobType ]: number } = {
   'activitypub-follow': 3,
   'video-file-import': 1,
   'video-file': 1,
+  'video-import': 1,
   'email': 5
 }
+const JOB_TTL: { [ id in JobType ]: number } = {
+  'activitypub-http-broadcast': 60000 * 10, // 10 minutes
+  'activitypub-http-unicast': 60000 * 10, // 10 minutes
+  'activitypub-http-fetcher': 60000 * 10, // 10 minutes
+  'activitypub-follow': 60000 * 10, // 10 minutes
+  'video-file-import': 1000 * 3600, // 1 hour
+  'video-file': 1000 * 3600 * 48, // 2 days, transcoding could be long
+  'video-import': 1000 * 3600 * 5, // 5 hours
+  'email': 60000 * 10 // 10 minutes
+}
 const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job
 const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds
-const JOB_REQUEST_TTL = 60000 * 10 // 10 minutes
 const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days
 
 // 1 hour
 let SCHEDULER_INTERVALS_MS = {
   badActorFollow: 60000 * 60, // 1 hour
   removeOldJobs: 60000 * 60, // 1 hour
-  updateVideos: 60000 // 1 minute
+  updateVideos: 60000, // 1 minute
+  youtubeDLUpdate: 60000 * 60 * 24 // 1 day
 }
 
 // ---------------------------------------------------------------------------
@@ -121,7 +135,10 @@ const CONFIG = {
     HOSTNAME: config.get<string>('database.hostname'),
     PORT: config.get<number>('database.port'),
     USERNAME: config.get<string>('database.username'),
-    PASSWORD: config.get<string>('database.password')
+    PASSWORD: config.get<string>('database.password'),
+    POOL: {
+      MAX: config.get<number>('database.pool.max')
+    }
   },
   REDIS: {
     HOSTNAME: config.has('redis.hostname') ? config.get<string>('redis.hostname') : null,
@@ -189,6 +206,13 @@ const CONFIG = {
       get '1080p' () { return config.get<boolean>('transcoding.resolutions.1080p') }
     }
   },
+  IMPORT: {
+    VIDEOS: {
+      HTTP: {
+        get ENABLED () { return config.get<boolean>('import.videos.http.enabled') }
+      }
+    }
+  },
   CACHE: {
     PREVIEWS: {
       get SIZE () { return config.get<number>('cache.previews.size') }
@@ -245,6 +269,9 @@ const CONSTRAINTS_FIELDS = {
       }
     }
   },
+  VIDEO_IMPORTS: {
+    URL: { min: 3, max: 2000 } // Length
+  },
   VIDEOS: {
     NAME: { min: 3, max: 120 }, // Length
     LANGUAGE: { min: 1, max: 10 }, // Length
@@ -259,7 +286,7 @@ const CONSTRAINTS_FIELDS = {
     },
     EXTNAME: [ '.mp4', '.ogv', '.webm' ],
     INFO_HASH: { min: 40, max: 40 }, // Length, info hash is 20 bytes length but we represent it in hexadecimal so 20 * 2
-    DURATION: { min: 1 }, // Number
+    DURATION: { min: 0 }, // Number
     TAGS: { min: 0, max: 5 }, // Number of total tags
     TAG: { min: 2, max: 30 }, // Length
     THUMBNAIL: { min: 2, max: 30 },
@@ -313,6 +340,11 @@ const VIDEO_RATE_TYPES: { [ id: string ]: VideoRateType } = {
   DISLIKE: 'dislike'
 }
 
+const FFMPEG_NICE: { [ id: string ]: number } = {
+  THUMBNAIL: 2, // 2 just for don't blocking servers
+  TRANSCODING: 15
+}
+
 const VIDEO_CATEGORIES = {
   1: 'Music',
   2: 'Films',
@@ -355,7 +387,14 @@ const VIDEO_PRIVACIES = {
 
 const VIDEO_STATES = {
   [VideoState.PUBLISHED]: 'Published',
-  [VideoState.TO_TRANSCODE]: 'To transcode'
+  [VideoState.TO_TRANSCODE]: 'To transcode',
+  [VideoState.TO_IMPORT]: 'To import'
+}
+
+const VIDEO_IMPORT_STATES = {
+  [VideoImportState.FAILED]: 'Failed',
+  [VideoImportState.PENDING]: 'Pending',
+  [VideoImportState.SUCCESS]: 'Success'
 }
 
 const VIDEO_MIMETYPE_EXT = {
@@ -553,6 +592,7 @@ export {
   ROUTE_CACHE_LIFETIME,
   SORTABLE_COLUMNS,
   FEEDS,
+  JOB_TTL,
   NSFW_POLICY_TYPES,
   STATIC_MAX_AGE,
   STATIC_PATHS,
@@ -567,8 +607,8 @@ export {
   VIDEO_RATE_TYPES,
   VIDEO_MIMETYPE_EXT,
   VIDEO_TRANSCODING_FPS,
+  FFMPEG_NICE,
   JOB_REQUEST_TIMEOUT,
-  JOB_REQUEST_TTL,
   USER_PASSWORD_RESET_LIFETIME,
   IMAGE_MIMETYPE_EXT,
   SCHEDULER_INTERVALS_MS,
@@ -576,6 +616,7 @@ export {
   RATES_LIMIT,
   VIDEO_EXT_MIMETYPE,
   JOB_COMPLETED_LIFETIME,
+  VIDEO_IMPORT_STATES,
   VIDEO_VIEW_LIFETIME,
   buildLanguages
 }
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 1a9ce5a61..0be752363 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -24,6 +24,7 @@ import { VideoTagModel } from '../models/video/video-tag'
 import { CONFIG } from './constants'
 import { ScheduleVideoUpdateModel } from '../models/video/schedule-video-update'
 import { VideoCaptionModel } from '../models/video/video-caption'
+import { VideoImportModel } from '../models/video/video-import'
 
 require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
 
@@ -32,6 +33,7 @@ const username = CONFIG.DATABASE.USERNAME
 const password = CONFIG.DATABASE.PASSWORD
 const host = CONFIG.DATABASE.HOSTNAME
 const port = CONFIG.DATABASE.PORT
+const poolMax = CONFIG.DATABASE.POOL.MAX
 
 const sequelizeTypescript = new SequelizeTypescript({
   database: dbname,
@@ -40,6 +42,9 @@ const sequelizeTypescript = new SequelizeTypescript({
   port,
   username,
   password,
+  pool: {
+    max: poolMax
+  },
   benchmark: isTestInstance(),
   isolationLevel: SequelizeTypescript.Transaction.ISOLATION_LEVELS.SERIALIZABLE,
   operatorsAliases: false,
@@ -77,7 +82,8 @@ async function initDatabaseModels (silent: boolean) {
     VideoTagModel,
     VideoModel,
     VideoCommentModel,
-    ScheduleVideoUpdateModel
+    ScheduleVideoUpdateModel,
+    VideoImportModel
   ])
 
   // Check extensions exist in the database