Add ability to import video with youtube-dl

16 files changed, 672 insertions, 41 deletions

diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts
new file mode 100644
index 000000000..9761cdbcf
--- /dev/null
+++ b/server/controllers/api/videos/import.ts
@@ -0,0 +1,151 @@
+import * as express from 'express'
+import { auditLoggerFactory } from '../../../helpers/audit-logger'
+import {
+  asyncMiddleware,
+  asyncRetryTransactionMiddleware,
+  authenticate,
+  videoImportAddValidator,
+  videoImportDeleteValidator
+} from '../../../middlewares'
+import { CONFIG, IMAGE_MIMETYPE_EXT, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE } from '../../../initializers'
+import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl'
+import { createReqFiles } from '../../../helpers/express-utils'
+import { logger } from '../../../helpers/logger'
+import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared'
+import { VideoModel } from '../../../models/video/video'
+import { getVideoActivityPubUrl } from '../../../lib/activitypub'
+import { TagModel } from '../../../models/video/tag'
+import { VideoImportModel } from '../../../models/video/video-import'
+import { JobQueue } from '../../../lib/job-queue/job-queue'
+import { processImage } from '../../../helpers/image-utils'
+import { join } from 'path'
+
+const auditLogger = auditLoggerFactory('video-imports')
+const videoImportsRouter = express.Router()
+
+const reqVideoFileImport = createReqFiles(
+  [ 'thumbnailfile', 'previewfile' ],
+  IMAGE_MIMETYPE_EXT,
+  {
+    thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR,
+    previewfile: CONFIG.STORAGE.PREVIEWS_DIR
+  }
+)
+
+videoImportsRouter.post('/imports',
+  authenticate,
+  reqVideoFileImport,
+  asyncMiddleware(videoImportAddValidator),
+  asyncRetryTransactionMiddleware(addVideoImport)
+)
+
+videoImportsRouter.delete('/imports/:id',
+  authenticate,
+  videoImportDeleteValidator,
+  asyncRetryTransactionMiddleware(deleteVideoImport)
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+  videoImportsRouter
+}
+
+// ---------------------------------------------------------------------------
+
+async function addVideoImport (req: express.Request, res: express.Response) {
+  const body: VideoImportCreate = req.body
+  const targetUrl = body.targetUrl
+
+  let youtubeDLInfo: YoutubeDLInfo
+  try {
+    youtubeDLInfo = await getYoutubeDLInfo(targetUrl)
+  } catch (err) {
+    logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
+
+    return res.status(400).json({
+      error: 'Cannot fetch remote information of this URL.'
+    }).end()
+  }
+
+  // Create video DB object
+  const videoData = {
+    name: body.name || youtubeDLInfo.name,
+    remote: false,
+    category: body.category || youtubeDLInfo.category,
+    licence: body.licence || youtubeDLInfo.licence,
+    language: undefined,
+    commentsEnabled: body.commentsEnabled || true,
+    waitTranscoding: body.waitTranscoding || false,
+    state: VideoState.TO_IMPORT,
+    nsfw: body.nsfw || youtubeDLInfo.nsfw || false,
+    description: body.description || youtubeDLInfo.description,
+    support: body.support || null,
+    privacy: body.privacy || VideoPrivacy.PRIVATE,
+    duration: 0, // duration will be set by the import job
+    channelId: res.locals.videoChannel.id
+  }
+  const video = new VideoModel(videoData)
+  video.url = getVideoActivityPubUrl(video)
+
+  // Process thumbnail file?
+  const thumbnailField = req.files['thumbnailfile']
+  let downloadThumbnail = true
+  if (thumbnailField) {
+    const thumbnailPhysicalFile = thumbnailField[ 0 ]
+    await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE)
+    downloadThumbnail = false
+  }
+
+  // Process preview file?
+  const previewField = req.files['previewfile']
+  let downloadPreview = true
+  if (previewField) {
+    const previewPhysicalFile = previewField[0]
+    await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE)
+    downloadPreview = false
+  }
+
+  const videoImport: VideoImportModel = await sequelizeTypescript.transaction(async t => {
+    const sequelizeOptions = { transaction: t }
+
+    // Save video object in database
+    const videoCreated = await video.save(sequelizeOptions)
+    videoCreated.VideoChannel = res.locals.videoChannel
+
+    // Set tags to the video
+    if (youtubeDLInfo.tags !== undefined) {
+      const tagInstances = await TagModel.findOrCreateTags(youtubeDLInfo.tags, t)
+
+      await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
+      videoCreated.Tags = tagInstances
+    }
+
+    // Create video import object in database
+    const videoImport = await VideoImportModel.create({
+      targetUrl,
+      state: VideoImportState.PENDING,
+      videoId: videoCreated.id
+    }, sequelizeOptions)
+
+    videoImport.Video = videoCreated
+
+    return videoImport
+  })
+
+  // Create job to import the video
+  const payload = {
+    type: 'youtube-dl' as 'youtube-dl',
+    videoImportId: videoImport.id,
+    thumbnailUrl: youtubeDLInfo.thumbnailUrl,
+    downloadThumbnail,
+    downloadPreview
+  }
+  await JobQueue.Instance.createJob({ type: 'video-import', payload })
+
+  return res.json(videoImport.toFormattedJSON())
+}
+
+async function deleteVideoImport (req: express.Request, res: express.Response) {
+  // TODO: delete video import
+}
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index e396ee6be..c9365da08 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -54,6 +54,7 @@ import { VideoFilter } from '../../../../shared/models/videos/video-query.type'
 import { createReqFiles, buildNSFWFilter } from '../../../helpers/express-utils'
 import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update'
 import { videoCaptionsRouter } from './captions'
+import { videoImportsRouter } from './import'
 
 const auditLogger = auditLoggerFactory('videos')
 const videosRouter = express.Router()
@@ -81,6 +82,7 @@ videosRouter.use('/', blacklistRouter)
 videosRouter.use('/', rateVideoRouter)
 videosRouter.use('/', videoCommentRouter)
 videosRouter.use('/', videoCaptionsRouter)
+videosRouter.use('/', videoImportsRouter)
 
 videosRouter.get('/categories', listVideoCategories)
 videosRouter.get('/licences', listVideoLicences)
@@ -160,7 +162,6 @@ async function addVideo (req: express.Request, res: express.Response) {
   const videoData = {
     name: videoInfo.name,
     remote: false,
-    extname: extname(videoPhysicalFile.filename),
     category: videoInfo.category,
     licence: videoInfo.licence,
     language: videoInfo.language,
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index d97bbd2a9..c6a350236 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -45,7 +45,7 @@ function isActivityPubVideoDurationValid (value: string) {
 }
 
 function sanitizeAndCheckVideoTorrentObject (video: any) {
-  if (video.type !== 'Video') return false
+  if (!video || video.type !== 'Video') return false
 
   if (!setValidRemoteTags(video)) return false
   if (!setValidRemoteVideoUrls(video)) return false
diff --git a/server/helpers/custom-validators/video-imports.ts b/server/helpers/custom-validators/video-imports.ts
new file mode 100644
index 000000000..36c0559fd
--- /dev/null
+++ b/server/helpers/custom-validators/video-imports.ts
@@ -0,0 +1,30 @@
+import 'express-validator'
+import 'multer'
+import * as validator from 'validator'
+import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers'
+import { exists } from './misc'
+
+function isVideoImportTargetUrlValid (url: string) {
+  const isURLOptions = {
+    require_host: true,
+    require_tld: true,
+    require_protocol: true,
+    require_valid_protocol: true,
+    protocols: [ 'http', 'https' ]
+  }
+
+  return exists(url) &&
+    validator.isURL('' + url, isURLOptions) &&
+    validator.isLength('' + url, CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL)
+}
+
+function isVideoImportStateValid (value: any) {
+  return exists(value) && VIDEO_IMPORT_STATES[ value ] !== undefined
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isVideoImportStateValid,
+  isVideoImportTargetUrlValid
+}
diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts
index 04a19a9c6..480c5b49e 100644
--- a/server/helpers/logger.ts
+++ b/server/helpers/logger.ts
@@ -22,7 +22,7 @@ function loggerReplacer (key: string, value: any) {
 }
 
 const consoleLoggerFormat = winston.format.printf(info => {
-  let additionalInfos = JSON.stringify(info.meta, loggerReplacer, 2)
+  let additionalInfos = JSON.stringify(info.meta || info.err, loggerReplacer, 2)
   if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = ''
   else additionalInfos = ' ' + additionalInfos
 
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
new file mode 100644
index 000000000..74d3e213b
--- /dev/null
+++ b/server/helpers/youtube-dl.ts
@@ -0,0 +1,142 @@
+import * as youtubeDL from 'youtube-dl'
+import { truncate } from 'lodash'
+import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers'
+import { join } from 'path'
+import * as crypto from 'crypto'
+import { logger } from './logger'
+
+export type YoutubeDLInfo = {
+  name: string
+  description: string
+  category: number
+  licence: number
+  nsfw: boolean
+  tags: string[]
+  thumbnailUrl: string
+}
+
+function getYoutubeDLInfo (url: string): Promise<YoutubeDLInfo> {
+  return new Promise<YoutubeDLInfo>((res, rej) => {
+    const options = [ '-j', '--flat-playlist' ]
+
+    youtubeDL.getInfo(url, options, (err, info) => {
+      if (err) return rej(err)
+
+      const obj = normalizeObject(info)
+
+      return res(buildVideoInfo(obj))
+    })
+  })
+}
+
+function downloadYoutubeDLVideo (url: string) {
+  const hash = crypto.createHash('sha256').update(url).digest('base64')
+  const path = join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4')
+
+  logger.info('Importing video %s', url)
+
+  const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
+
+  return new Promise<string>((res, rej) => {
+    youtubeDL.exec(url, options, async (err, output) => {
+      if (err) return rej(err)
+
+      return res(path)
+    })
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  downloadYoutubeDLVideo,
+  getYoutubeDLInfo
+}
+
+// ---------------------------------------------------------------------------
+
+function normalizeObject (obj: any) {
+  const newObj: any = {}
+
+  for (const key of Object.keys(obj)) {
+    // Deprecated key
+    if (key === 'resolution') continue
+
+    const value = obj[key]
+
+    if (typeof value === 'string') {
+      newObj[key] = value.normalize()
+    } else {
+      newObj[key] = value
+    }
+  }
+
+  return newObj
+}
+
+function buildVideoInfo (obj: any) {
+  return {
+    name: titleTruncation(obj.title),
+    description: descriptionTruncation(obj.description),
+    category: getCategory(obj.categories),
+    licence: getLicence(obj.license),
+    nsfw: isNSFW(obj),
+    tags: getTags(obj.tags),
+    thumbnailUrl: obj.thumbnail || undefined
+  }
+}
+
+function titleTruncation (title: string) {
+  return truncate(title, {
+    'length': CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
+    'separator': /,? +/,
+    'omission': ' […]'
+  })
+}
+
+function descriptionTruncation (description: string) {
+  if (!description) return undefined
+
+  return truncate(description, {
+    'length': CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max,
+    'separator': /,? +/,
+    'omission': ' […]'
+  })
+}
+
+function isNSFW (info: any) {
+  return info.age_limit && info.age_limit >= 16
+}
+
+function getTags (tags: any) {
+  if (Array.isArray(tags) === false) return []
+
+  return tags
+    .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
+    .map(t => t.normalize())
+    .slice(0, 5)
+}
+
+function getLicence (licence: string) {
+  if (!licence) return undefined
+
+  if (licence.indexOf('Creative Commons Attribution licence') !== -1) return 1
+
+  return undefined
+}
+
+function getCategory (categories: string[]) {
+  if (!categories) return undefined
+
+  const categoryString = categories[0]
+  if (!categoryString || typeof categoryString !== 'string') return undefined
+
+  if (categoryString === 'News & Politics') return 11
+
+  for (const key of Object.keys(VIDEO_CATEGORIES)) {
+    const category = VIDEO_CATEGORIES[key]
+    if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10)
+  }
+
+  return undefined
+}
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index bec343bb7..fdd772d84 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -8,6 +8,7 @@ import { VideoPrivacy } from '../../shared/models/videos'
 import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils'
 import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
 import { invert } from 'lodash'
+import { VideoImportState } from '../../shared/models/videos/video-import-state.enum'
 
 // Use a variable to reload the configuration if we need
 let config: IConfig = require('config')
@@ -85,6 +86,7 @@ const JOB_ATTEMPTS: { [ id in JobType ]: number } = {
   'activitypub-follow': 5,
   'video-file-import': 1,
   'video-file': 1,
+  'video-import': 1,
   'email': 5
 }
 const JOB_CONCURRENCY: { [ id in JobType ]: number } = {
@@ -94,6 +96,7 @@ const JOB_CONCURRENCY: { [ id in JobType ]: number } = {
   'activitypub-follow': 3,
   'video-file-import': 1,
   'video-file': 1,
+  'video-import': 1,
   'email': 5
 }
 const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job
@@ -248,6 +251,9 @@ const CONSTRAINTS_FIELDS = {
       }
     }
   },
+  VIDEO_IMPORTS: {
+    URL: { min: 3, max: 2000 } // Length
+  },
   VIDEOS: {
     NAME: { min: 3, max: 120 }, // Length
     LANGUAGE: { min: 1, max: 10 }, // Length
@@ -262,7 +268,7 @@ const CONSTRAINTS_FIELDS = {
     },
     EXTNAME: [ '.mp4', '.ogv', '.webm' ],
     INFO_HASH: { min: 40, max: 40 }, // Length, info hash is 20 bytes length but we represent it in hexadecimal so 20 * 2
-    DURATION: { min: 1 }, // Number
+    DURATION: { min: 0 }, // Number
     TAGS: { min: 0, max: 5 }, // Number of total tags
     TAG: { min: 2, max: 30 }, // Length
     THUMBNAIL: { min: 2, max: 30 },
@@ -363,7 +369,14 @@ const VIDEO_PRIVACIES = {
 
 const VIDEO_STATES = {
   [VideoState.PUBLISHED]: 'Published',
-  [VideoState.TO_TRANSCODE]: 'To transcode'
+  [VideoState.TO_TRANSCODE]: 'To transcode',
+  [VideoState.TO_IMPORT]: 'To import'
+}
+
+const VIDEO_IMPORT_STATES = {
+  [VideoImportState.FAILED]: 'Failed',
+  [VideoImportState.PENDING]: 'Pending',
+  [VideoImportState.SUCCESS]: 'Success'
 }
 
 const VIDEO_MIMETYPE_EXT = {
@@ -585,6 +598,7 @@ export {
   RATES_LIMIT,
   VIDEO_EXT_MIMETYPE,
   JOB_COMPLETED_LIFETIME,
+  VIDEO_IMPORT_STATES,
   VIDEO_VIEW_LIFETIME,
   buildLanguages
 }
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 21c083084..0be752363 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -24,6 +24,7 @@ import { VideoTagModel } from '../models/video/video-tag'
 import { CONFIG } from './constants'
 import { ScheduleVideoUpdateModel } from '../models/video/schedule-video-update'
 import { VideoCaptionModel } from '../models/video/video-caption'
+import { VideoImportModel } from '../models/video/video-import'
 
 require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
 
@@ -81,7 +82,8 @@ async function initDatabaseModels (silent: boolean) {
     VideoTagModel,
     VideoModel,
     VideoCommentModel,
-    ScheduleVideoUpdateModel
+    ScheduleVideoUpdateModel,
+    VideoImportModel
   ])
 
   // Check extensions exist in the database
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
new file mode 100644
index 000000000..2f219e986
--- /dev/null
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -0,0 +1,129 @@
+import * as Bull from 'bull'
+import { logger } from '../../../helpers/logger'
+import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl'
+import { VideoImportModel } from '../../../models/video/video-import'
+import { VideoImportState } from '../../../../shared/models/videos'
+import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
+import { extname, join } from 'path'
+import { VideoFileModel } from '../../../models/video/video-file'
+import { renamePromise, statPromise, unlinkPromise } from '../../../helpers/core-utils'
+import { CONFIG, sequelizeTypescript } from '../../../initializers'
+import { doRequestAndSaveToFile } from '../../../helpers/requests'
+import { VideoState } from '../../../../shared'
+import { JobQueue } from '../index'
+import { federateVideoIfNeeded } from '../../activitypub'
+
+export type VideoImportPayload = {
+  type: 'youtube-dl'
+  videoImportId: number
+  thumbnailUrl: string
+  downloadThumbnail: boolean
+  downloadPreview: boolean
+}
+
+async function processVideoImport (job: Bull.Job) {
+  const payload = job.data as VideoImportPayload
+  logger.info('Processing video import in job %d.', job.id)
+
+  const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId)
+  if (!videoImport) throw new Error('Cannot import video %s: the video import entry does not exist anymore.')
+
+  let tempVideoPath: string
+  try {
+    // Download video from youtubeDL
+    tempVideoPath = await downloadYoutubeDLVideo(videoImport.targetUrl)
+
+    // Get information about this video
+    const { videoFileResolution } = await getVideoFileResolution(tempVideoPath)
+    const fps = await getVideoFileFPS(tempVideoPath)
+    const stats = await statPromise(tempVideoPath)
+    const duration = await getDurationFromVideoFile(tempVideoPath)
+
+    // Create video file object in database
+    const videoFileData = {
+      extname: extname(tempVideoPath),
+      resolution: videoFileResolution,
+      size: stats.size,
+      fps,
+      videoId: videoImport.videoId
+    }
+    const videoFile = new VideoFileModel(videoFileData)
+
+    // Move file
+    const destination = join(CONFIG.STORAGE.VIDEOS_DIR, videoImport.Video.getVideoFilename(videoFile))
+    await renamePromise(tempVideoPath, destination)
+
+    // Process thumbnail
+    if (payload.downloadThumbnail) {
+      if (payload.thumbnailUrl) {
+        const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName())
+        await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destThumbnailPath)
+      } else {
+        await videoImport.Video.createThumbnail(videoFile)
+      }
+    }
+
+    // Process preview
+    if (payload.downloadPreview) {
+      if (payload.thumbnailUrl) {
+        const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName())
+        await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destPreviewPath)
+      } else {
+        await videoImport.Video.createPreview(videoFile)
+      }
+    }
+
+    // Create torrent
+    await videoImport.Video.createTorrentAndSetInfoHash(videoFile)
+
+    const videoImportUpdated: VideoImportModel = await sequelizeTypescript.transaction(async t => {
+      await videoFile.save({ transaction: t })
+
+      // Update video DB object
+      videoImport.Video.duration = duration
+      videoImport.Video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED
+      const videoUpdated = await videoImport.Video.save({ transaction: t })
+
+      // Now we can federate the video
+      await federateVideoIfNeeded(videoImport.Video, true, t)
+
+      // Update video import object
+      videoImport.state = VideoImportState.SUCCESS
+      const videoImportUpdated = await videoImport.save({ transaction: t })
+
+      logger.info('Video %s imported.', videoImport.targetUrl)
+
+      videoImportUpdated.Video = videoUpdated
+      return videoImportUpdated
+    })
+
+    // Create transcoding jobs?
+    if (videoImportUpdated.Video.state === VideoState.TO_TRANSCODE) {
+      // Put uuid because we don't have id auto incremented for now
+      const dataInput = {
+        videoUUID: videoImportUpdated.Video.uuid,
+        isNewVideo: true
+      }
+
+      await JobQueue.Instance.createJob({ type: 'video-file', payload: dataInput })
+    }
+
+  } catch (err) {
+    try {
+      if (tempVideoPath) await unlinkPromise(tempVideoPath)
+    } catch (errUnlink) {
+      logger.error('Cannot cleanup files after a video import error.', { err: errUnlink })
+    }
+
+    videoImport.state = VideoImportState.FAILED
+    await videoImport.save()
+
+    throw err
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  processVideoImport
+}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 8ff0c169e..2e14867f2 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -9,6 +9,7 @@ import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './
 import { EmailPayload, processEmail } from './handlers/email'
 import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file'
 import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow'
+import { processVideoImport, VideoImportPayload } from './handlers/video-import'
 
 type CreateJobArgument =
   { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -17,7 +18,8 @@ type CreateJobArgument =
   { type: 'activitypub-follow', payload: ActivitypubFollowPayload } |
   { type: 'video-file-import', payload: VideoFileImportPayload } |
   { type: 'video-file', payload: VideoFilePayload } |
-  { type: 'email', payload: EmailPayload }
+  { type: 'email', payload: EmailPayload } |
+  { type: 'video-import', payload: VideoImportPayload }
 
 const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
   'activitypub-http-broadcast': processActivityPubHttpBroadcast,
@@ -26,7 +28,8 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
   'activitypub-follow': processActivityPubFollow,
   'video-file-import': processVideoFileImport,
   'video-file': processVideoFile,
-  'email': processEmail
+  'email': processEmail,
+  'video-import': processVideoImport
 }
 
 const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = {
@@ -43,7 +46,8 @@ const jobTypes: JobType[] = [
   'activitypub-http-unicast',
   'email',
   'video-file',
-  'video-file-import'
+  'video-file-import',
+  'video-import'
 ]
 
 class JobQueue {
diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts
index e3f0f5963..c5400c8f5 100644
--- a/server/middlewares/validators/index.ts
+++ b/server/middlewares/validators/index.ts
@@ -11,3 +11,4 @@ export * from './video-blacklist'
 export * from './video-channels'
 export * from './webfinger'
 export * from './search'
+export * from './video-imports'
diff --git a/server/middlewares/validators/video-imports.ts b/server/middlewares/validators/video-imports.ts
new file mode 100644
index 000000000..0ba759ff0
--- /dev/null
+++ b/server/middlewares/validators/video-imports.ts
@@ -0,0 +1,51 @@
+import * as express from 'express'
+import { body, param } from 'express-validator/check'
+import { isIdValid } from '../../helpers/custom-validators/misc'
+import { logger } from '../../helpers/logger'
+import { areValidationErrors } from './utils'
+import { getCommonVideoAttributes } from './videos'
+import { isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports'
+import { cleanUpReqFiles } from '../../helpers/utils'
+import { isVideoChannelOfAccountExist, isVideoNameValid } from '../../helpers/custom-validators/videos'
+
+const videoImportAddValidator = getCommonVideoAttributes().concat([
+  body('targetUrl').custom(isVideoImportTargetUrlValid).withMessage('Should have a valid video import target URL'),
+  body('channelId')
+    .toInt()
+    .custom(isIdValid).withMessage('Should have correct video channel id'),
+  body('name')
+    .optional()
+    .custom(isVideoNameValid).withMessage('Should have a valid name'),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    logger.debug('Checking videoImportAddValidator parameters', { parameters: req.body })
+
+    const user = res.locals.oauth.token.User
+
+    if (areValidationErrors(req, res)) return cleanUpReqFiles(req)
+    if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req)
+
+    return next()
+  }
+])
+
+const videoImportDeleteValidator = [
+  param('id').custom(isIdValid).not().isEmpty().withMessage('Should have a valid id'),
+
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    logger.debug('Checking videoImportDeleteValidator parameters', { parameters: req.body })
+
+    if (areValidationErrors(req, res)) return
+
+    return next()
+  }
+]
+
+// ---------------------------------------------------------------------------
+
+export {
+  videoImportAddValidator,
+  videoImportDeleteValidator
+}
+
+// ---------------------------------------------------------------------------
diff --git a/server/middlewares/validators/videos.ts b/server/middlewares/validators/videos.ts
index 9357c1e39..c812d4677 100644
--- a/server/middlewares/validators/videos.ts
+++ b/server/middlewares/validators/videos.ts
@@ -223,36 +223,6 @@ const videosShareValidator = [
   }
 ]
 
-// ---------------------------------------------------------------------------
-
-export {
-  videosAddValidator,
-  videosUpdateValidator,
-  videosGetValidator,
-  videosRemoveValidator,
-  videosShareValidator,
-
-  videoAbuseReportValidator,
-
-  videoRateValidator
-}
-
-// ---------------------------------------------------------------------------
-
-function areErrorsInScheduleUpdate (req: express.Request, res: express.Response) {
-  if (req.body.scheduleUpdate) {
-    if (!req.body.scheduleUpdate.updateAt) {
-      res.status(400)
-         .json({ error: 'Schedule update at is mandatory.' })
-         .end()
-
-      return true
-    }
-  }
-
-  return false
-}
-
 function getCommonVideoAttributes () {
   return [
     body('thumbnailfile')
@@ -319,3 +289,35 @@ function getCommonVideoAttributes () {
       .custom(isScheduleVideoUpdatePrivacyValid).withMessage('Should have correct schedule update privacy')
   ] as (ValidationChain | express.Handler)[]
 }
+
+// ---------------------------------------------------------------------------
+
+export {
+  videosAddValidator,
+  videosUpdateValidator,
+  videosGetValidator,
+  videosRemoveValidator,
+  videosShareValidator,
+
+  videoAbuseReportValidator,
+
+  videoRateValidator,
+
+  getCommonVideoAttributes
+}
+
+// ---------------------------------------------------------------------------
+
+function areErrorsInScheduleUpdate (req: express.Request, res: express.Response) {
+  if (req.body.scheduleUpdate) {
+    if (!req.body.scheduleUpdate.updateAt) {
+      res.status(400)
+         .json({ error: 'Schedule update at is mandatory.' })
+         .end()
+
+      return true
+    }
+  }
+
+  return false
+}
diff --git a/server/models/account/account.ts b/server/models/account/account.ts
index d674d8d22..66f5dcf2e 100644
--- a/server/models/account/account.ts
+++ b/server/models/account/account.ts
@@ -16,7 +16,6 @@ import {
 } from 'sequelize-typescript'
 import { Account } from '../../../shared/models/actors'
 import { isAccountDescriptionValid } from '../../helpers/custom-validators/accounts'
-import { logger } from '../../helpers/logger'
 import { sendDeleteActor } from '../../lib/activitypub/send'
 import { ActorModel } from '../activitypub/actor'
 import { ApplicationModel } from '../application/application'
diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts
new file mode 100644
index 000000000..89eeafd6a
--- /dev/null
+++ b/server/models/video/video-import.ts
@@ -0,0 +1,105 @@
+import {
+  AllowNull,
+  BelongsTo,
+  Column,
+  CreatedAt,
+  DataType,
+  Default,
+  DefaultScope,
+  ForeignKey,
+  Is,
+  Model,
+  Table,
+  UpdatedAt
+} from 'sequelize-typescript'
+import { CONSTRAINTS_FIELDS } from '../../initializers'
+import { throwIfNotValid } from '../utils'
+import { VideoModel } from './video'
+import { isVideoImportStateValid, isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports'
+import { VideoImport, VideoImportState } from '../../../shared'
+import { VideoChannelModel } from './video-channel'
+import { AccountModel } from '../account/account'
+
+@DefaultScope({
+  include: [
+    {
+      model: () => VideoModel,
+      required: true,
+      include: [
+        {
+          model: () => VideoChannelModel,
+          required: true,
+          include: [
+            {
+              model: () => AccountModel,
+              required: true
+            }
+          ]
+        }
+      ]
+    }
+  ]
+})
+
+@Table({
+  tableName: 'videoImport',
+  indexes: [
+    {
+      fields: [ 'videoId' ],
+      unique: true
+    }
+  ]
+})
+export class VideoImportModel extends Model<VideoImportModel> {
+  @CreatedAt
+  createdAt: Date
+
+  @UpdatedAt
+  updatedAt: Date
+
+  @AllowNull(false)
+  @Is('VideoImportTargetUrl', value => throwIfNotValid(value, isVideoImportTargetUrlValid, 'targetUrl'))
+  @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max))
+  targetUrl: string
+
+  @AllowNull(false)
+  @Default(null)
+  @Is('VideoImportState', value => throwIfNotValid(value, isVideoImportStateValid, 'state'))
+  @Column
+  state: VideoImportState
+
+  @AllowNull(true)
+  @Default(null)
+  @Column(DataType.TEXT)
+  error: string
+
+  @ForeignKey(() => VideoModel)
+  @Column
+  videoId: number
+
+  @BelongsTo(() => VideoModel, {
+    foreignKey: {
+      allowNull: false
+    },
+    onDelete: 'CASCADE'
+  })
+  Video: VideoModel
+
+  static loadAndPopulateVideo (id: number) {
+    return VideoImportModel.findById(id)
+  }
+
+  toFormattedJSON (): VideoImport {
+    const videoFormatOptions = {
+      additionalAttributes: { state: true, waitTranscoding: true, scheduledUpdate: true }
+    }
+    const video = Object.assign(this.Video.toFormattedJSON(videoFormatOptions), {
+      tags: this.Video.Tags.map(t => t.name)
+    })
+
+    return {
+      targetUrl: this.targetUrl,
+      video
+    }
+  }
+}
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index a6c4620b2..459fcb31e 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -377,7 +377,7 @@ type AvailableForListOptions = {
     include: [
       {
         model: () => VideoFileModel.unscoped(),
-        required: true
+        required: false
       }
     ]
   },