12 files changed, 68 insertions, 148 deletions

diff --git a/server/lib/job-queue/handlers/activitypub-follow.ts b/server/lib/job-queue/handlers/activitypub-follow.ts
index 4a7cda0a2..7034c10d0 100644
--- a/server/lib/job-queue/handlers/activitypub-follow.ts
+++ b/server/lib/job-queue/handlers/activitypub-follow.ts
@@ -11,13 +11,7 @@ import { ActorModel } from '../../../models/activitypub/actor'
 import { Notifier } from '../../notifier'
 import { sequelizeTypescript } from '../../../initializers/database'
 import { MActor, MActorFollowActors, MActorFull } from '../../../typings/models'
-
-export type ActivitypubFollowPayload = {
-  followerActorId: number
-  name: string
-  host: string
-  isAutoFollow?: boolean
-}
+import { ActivitypubFollowPayload } from '@shared/models'
 
 async function processActivityPubFollow (job: Bull.Job) {
   const payload = job.data as ActivitypubFollowPayload
@@ -34,6 +28,11 @@ async function processActivityPubFollow (job: Bull.Job) {
     targetActor = await getOrCreateActorAndServerAndModel(actorUrl, 'all')
   }
 
+  if (payload.assertIsChannel && !targetActor.VideoChannel) {
+    logger.warn('Do not follow %s@%s because it is not a channel.', payload.name, host)
+    return
+  }
+
   const fromActor = await ActorModel.load(payload.followerActorId)
 
   return retryTransactionWrapper(follow, fromActor, targetActor, payload.isAutoFollow)
diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
index 0ff7b44a0..e4d3dbbff 100644
--- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
@@ -5,12 +5,7 @@ import { doRequest } from '../../../helpers/requests'
 import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
 import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers/constants'
 import { ActorFollowScoreCache } from '../../files-cache'
-
-export type ActivitypubHttpBroadcastPayload = {
-  uris: string[]
-  signatureActorId?: number
-  body: any
-}
+import { ActivitypubHttpBroadcastPayload } from '@shared/models'
 
 async function processActivityPubHttpBroadcast (job: Bull.Job) {
   logger.info('Processing ActivityPub broadcast in job %d.', job.id)
diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
index 0182c5169..524aadc27 100644
--- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
@@ -5,22 +5,15 @@ import { processActivities } from '../../activitypub/process'
 import { addVideoComments } from '../../activitypub/video-comments'
 import { crawlCollectionPage } from '../../activitypub/crawl'
 import { VideoModel } from '../../../models/video/video'
-import { addVideoShares, createRates } from '../../activitypub'
+import { addVideoShares } from '../../activitypub/share'
+import { createRates } from '../../activitypub/video-rates'
 import { createAccountPlaylists } from '../../activitypub/playlist'
 import { AccountModel } from '../../../models/account/account'
 import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
 import { VideoShareModel } from '../../../models/video/video-share'
 import { VideoCommentModel } from '../../../models/video/video-comment'
 import { MAccountDefault, MVideoFullLight } from '../../../typings/models'
-
-type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments' | 'account-playlists'
-
-export type ActivitypubHttpFetcherPayload = {
-  uri: string
-  type: FetchType
-  videoId?: number
-  accountId?: number
-}
+import { ActivitypubHttpFetcherPayload, FetchType } from '@shared/models'
 
 async function processActivityPubHttpFetcher (job: Bull.Job) {
   logger.info('Processing ActivityPub fetcher in job %d.', job.id)
diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts
index c70ce3be9..b65eeb677 100644
--- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-unicast.ts
@@ -4,12 +4,7 @@ import { doRequest } from '../../../helpers/requests'
 import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
 import { JOB_REQUEST_TIMEOUT } from '../../../initializers/constants'
 import { ActorFollowScoreCache } from '../../files-cache'
-
-export type ActivitypubHttpUnicastPayload = {
-  uri: string
-  signatureActorId?: number
-  body: any
-}
+import { ActivitypubHttpUnicastPayload } from '@shared/models'
 
 async function processActivityPubHttpUnicast (job: Bull.Job) {
   logger.info('Processing ActivityPub unicast in job %d.', job.id)
diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts
index 4d6c38cfa..666e56868 100644
--- a/server/lib/job-queue/handlers/activitypub-refresher.ts
+++ b/server/lib/job-queue/handlers/activitypub-refresher.ts
@@ -1,14 +1,12 @@
 import * as Bull from 'bull'
 import { logger } from '../../../helpers/logger'
 import { fetchVideoByUrl } from '../../../helpers/video'
-import { refreshActorIfNeeded, refreshVideoIfNeeded, refreshVideoPlaylistIfNeeded } from '../../activitypub'
+import { refreshActorIfNeeded } from '../../activitypub/actor'
+import { refreshVideoIfNeeded } from '../../activitypub/videos'
 import { ActorModel } from '../../../models/activitypub/actor'
 import { VideoPlaylistModel } from '../../../models/video/video-playlist'
-
-export type RefreshPayload = {
-  type: 'video' | 'video-playlist' | 'actor'
-  url: string
-}
+import { RefreshPayload } from '@shared/models'
+import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlist'
 
 async function refreshAPObject (job: Bull.Job) {
   const payload = job.data as RefreshPayload
diff --git a/server/lib/job-queue/handlers/email.ts b/server/lib/job-queue/handlers/email.ts
index 62701222c..3157731e2 100644
--- a/server/lib/job-queue/handlers/email.ts
+++ b/server/lib/job-queue/handlers/email.ts
@@ -1,8 +1,7 @@
 import * as Bull from 'bull'
 import { logger } from '../../../helpers/logger'
-import { Emailer, SendEmailOptions } from '../../emailer'
-
-export type EmailPayload = SendEmailOptions
+import { Emailer } from '../../emailer'
+import { EmailPayload } from '@shared/models'
 
 async function processEmail (job: Bull.Job) {
   const payload = job.data as EmailPayload
diff --git a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts
index d3bde6e6a..bcb49a731 100644
--- a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts
+++ b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts
@@ -1,11 +1,12 @@
 import { buildSignedActivity } from '../../../../helpers/activitypub'
-import { getServerActor } from '../../../../helpers/utils'
 import { ActorModel } from '../../../../models/activitypub/actor'
-import { sha256 } from '../../../../helpers/core-utils'
-import { HTTP_SIGNATURE } from '../../../../initializers/constants'
+import { ACTIVITY_PUB, HTTP_SIGNATURE } from '../../../../initializers/constants'
 import { MActor } from '../../../../typings/models'
+import { getServerActor } from '@server/models/application/application'
+import { buildDigest } from '@server/helpers/peertube-crypto'
+import { ContextType } from '@shared/models/activitypub/context'
 
-type Payload = { body: any, signatureActorId?: number }
+type Payload = { body: any, contextType?: ContextType, signatureActorId?: number }
 
 async function computeBody (payload: Payload) {
   let body = payload.body
@@ -13,7 +14,7 @@ async function computeBody (payload: Payload) {
   if (payload.signatureActorId) {
     const actorSignature = await ActorModel.load(payload.signatureActorId)
     if (!actorSignature) throw new Error('Unknown signature actor id.')
-    body = await buildSignedActivity(actorSignature, payload.body)
+    body = await buildSignedActivity(actorSignature, payload.body, payload.contextType)
   }
 
   return body
@@ -42,18 +43,13 @@ async function buildSignedRequestOptions (payload: Payload) {
 
 function buildGlobalHeaders (body: any) {
   return {
-    'Digest': buildDigest(body)
+    'Digest': buildDigest(body),
+    'Content-Type': 'application/activity+json',
+    'Accept': ACTIVITY_PUB.ACCEPT_HEADER
   }
 }
 
-function buildDigest (body: any) {
-  const rawBody = typeof body === 'string' ? body : JSON.stringify(body)
-
-  return 'SHA-256=' + sha256(rawBody, 'base64')
-}
-
 export {
-  buildDigest,
   buildGlobalHeaders,
   computeBody,
   buildSignedRequestOptions
diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts
index 99c991e72..ae11f1de3 100644
--- a/server/lib/job-queue/handlers/video-file-import.ts
+++ b/server/lib/job-queue/handlers/video-file-import.ts
@@ -9,11 +9,7 @@ import { extname } from 'path'
 import { MVideoFile, MVideoWithFile } from '@server/typings/models'
 import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
 import { getVideoFilePath } from '@server/lib/video-paths'
-
-export type VideoFileImportPayload = {
-  videoUUID: string,
-  filePath: string
-}
+import { VideoFileImportPayload } from '@shared/models'
 
 async function processVideoFileImport (job: Bull.Job) {
   const payload = job.data as VideoFileImportPayload
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts
index 1fca17584..ad549c6fc 100644
--- a/server/lib/job-queue/handlers/video-import.ts
+++ b/server/lib/job-queue/handlers/video-import.ts
@@ -7,9 +7,8 @@ import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } fro
 import { extname } from 'path'
 import { VideoFileModel } from '../../../models/video/video-file'
 import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
-import { VideoState } from '../../../../shared'
-import { JobQueue } from '../index'
-import { federateVideoIfNeeded } from '../../activitypub'
+import { VideoImportPayload, VideoImportTorrentPayload, VideoImportYoutubeDLPayload, VideoState } from '../../../../shared'
+import { federateVideoIfNeeded } from '../../activitypub/videos'
 import { VideoModel } from '../../../models/video/video'
 import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
 import { getSecureTorrentName } from '../../../helpers/utils'
@@ -17,27 +16,12 @@ import { move, remove, stat } from 'fs-extra'
 import { Notifier } from '../../notifier'
 import { CONFIG } from '../../../initializers/config'
 import { sequelizeTypescript } from '../../../initializers/database'
-import { createVideoMiniatureFromUrl, generateVideoMiniature } from '../../thumbnail'
+import { generateVideoMiniature } from '../../thumbnail'
 import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
 import { MThumbnail } from '../../../typings/models/video/thumbnail'
 import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import'
 import { getVideoFilePath } from '@server/lib/video-paths'
-
-type VideoImportYoutubeDLPayload = {
-  type: 'youtube-dl'
-  videoImportId: number
-
-  thumbnailUrl: string
-  downloadThumbnail: boolean
-  downloadPreview: boolean
-}
-
-type VideoImportTorrentPayload = {
-  type: 'magnet-uri' | 'torrent-file'
-  videoImportId: number
-}
-
-export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload
+import { addOptimizeOrMergeAudioJob } from '@server/helpers/video'
 
 async function processVideoImport (job: Bull.Job) {
   const payload = job.data as VideoImportPayload
@@ -62,9 +46,6 @@ async function processTorrentImport (job: Bull.Job, payload: VideoImportTorrentP
   const options = {
     videoImportId: payload.videoImportId,
 
-    downloadThumbnail: false,
-    downloadPreview: false,
-
     generateThumbnail: true,
     generatePreview: true
   }
@@ -82,15 +63,11 @@ async function processYoutubeDLImport (job: Bull.Job, payload: VideoImportYoutub
   const options = {
     videoImportId: videoImport.id,
 
-    downloadThumbnail: payload.downloadThumbnail,
-    downloadPreview: payload.downloadPreview,
-    thumbnailUrl: payload.thumbnailUrl,
-
-    generateThumbnail: false,
-    generatePreview: false
+    generateThumbnail: payload.generateThumbnail,
+    generatePreview: payload.generatePreview
   }
 
-  return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl, VIDEO_IMPORT_TIMEOUT), videoImport, options)
+  return processFile(() => downloadYoutubeDLVideo(videoImport.targetUrl, payload.fileExt, VIDEO_IMPORT_TIMEOUT), videoImport, options)
 }
 
 async function getVideoImportOrDie (videoImportId: number) {
@@ -105,10 +82,6 @@ async function getVideoImportOrDie (videoImportId: number) {
 type ProcessFileOptions = {
   videoImportId: number
 
-  downloadThumbnail: boolean
-  downloadPreview: boolean
-  thumbnailUrl?: string
-
   generateThumbnail: boolean
   generatePreview: boolean
 }
@@ -153,17 +126,13 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
 
     // Process thumbnail
     let thumbnailModel: MThumbnail
-    if (options.downloadThumbnail && options.thumbnailUrl) {
-      thumbnailModel = await createVideoMiniatureFromUrl(options.thumbnailUrl, videoImportWithFiles.Video, ThumbnailType.MINIATURE)
-    } else if (options.generateThumbnail || options.downloadThumbnail) {
+    if (options.generateThumbnail) {
       thumbnailModel = await generateVideoMiniature(videoImportWithFiles.Video, videoFile, ThumbnailType.MINIATURE)
     }
 
     // Process preview
     let previewModel: MThumbnail
-    if (options.downloadPreview && options.thumbnailUrl) {
-      previewModel = await createVideoMiniatureFromUrl(options.thumbnailUrl, videoImportWithFiles.Video, ThumbnailType.PREVIEW)
-    } else if (options.generatePreview || options.downloadPreview) {
+    if (options.generatePreview) {
       previewModel = await generateVideoMiniature(videoImportWithFiles.Video, videoFile, ThumbnailType.PREVIEW)
     }
 
@@ -214,14 +183,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
 
     // Create transcoding jobs?
     if (video.state === VideoState.TO_TRANSCODE) {
-      // Put uuid because we don't have id auto incremented for now
-      const dataInput = {
-        type: 'optimize' as 'optimize',
-        videoUUID: videoImportUpdated.Video.uuid,
-        isNewVideo: true
-      }
-
-      await JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput })
+      await addOptimizeOrMergeAudioJob(videoImportUpdated.Video, videoFile)
     }
 
   } catch (err) {
diff --git a/server/lib/job-queue/handlers/video-redundancy.ts b/server/lib/job-queue/handlers/video-redundancy.ts
new file mode 100644
index 000000000..6296dab05
--- /dev/null
+++ b/server/lib/job-queue/handlers/video-redundancy.ts
@@ -0,0 +1,17 @@
+import * as Bull from 'bull'
+import { logger } from '../../../helpers/logger'
+import { VideosRedundancyScheduler } from '@server/lib/schedulers/videos-redundancy-scheduler'
+import { VideoRedundancyPayload } from '@shared/models'
+
+async function processVideoRedundancy (job: Bull.Job) {
+  const payload = job.data as VideoRedundancyPayload
+  logger.info('Processing video redundancy in job %d.', job.id)
+
+  return VideosRedundancyScheduler.Instance.createManualRedundancy(payload.videoId)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  processVideoRedundancy
+}
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts
index 39b9fac98..46d52e1cf 100644
--- a/server/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/lib/job-queue/handlers/video-transcoding.ts
@@ -1,48 +1,22 @@
 import * as Bull from 'bull'
-import { VideoResolution } from '../../../../shared'
+import {
+  MergeAudioTranscodingPayload,
+  NewResolutionTranscodingPayload,
+  OptimizeTranscodingPayload,
+  VideoTranscodingPayload
+} from '../../../../shared'
 import { logger } from '../../../helpers/logger'
 import { VideoModel } from '../../../models/video/video'
 import { JobQueue } from '../job-queue'
-import { federateVideoIfNeeded } from '../../activitypub'
+import { federateVideoIfNeeded } from '../../activitypub/videos'
 import { retryTransactionWrapper } from '../../../helpers/database-utils'
-import { sequelizeTypescript } from '../../../initializers'
-import * as Bluebird from 'bluebird'
+import { sequelizeTypescript } from '../../../initializers/database'
 import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils'
 import { generateHlsPlaylist, mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewResolution } from '../../video-transcoding'
 import { Notifier } from '../../notifier'
 import { CONFIG } from '../../../initializers/config'
 import { MVideoFullLight, MVideoUUID, MVideoWithFile } from '@server/typings/models'
 
-interface BaseTranscodingPayload {
-  videoUUID: string
-  isNewVideo?: boolean
-}
-
-interface HLSTranscodingPayload extends BaseTranscodingPayload {
-  type: 'hls'
-  isPortraitMode?: boolean
-  resolution: VideoResolution
-  copyCodecs: boolean
-}
-
-interface NewResolutionTranscodingPayload extends BaseTranscodingPayload {
-  type: 'new-resolution'
-  isPortraitMode?: boolean
-  resolution: VideoResolution
-}
-
-interface MergeAudioTranscodingPayload extends BaseTranscodingPayload {
-  type: 'merge-audio'
-  resolution: VideoResolution
-}
-
-interface OptimizeTranscodingPayload extends BaseTranscodingPayload {
-  type: 'optimize'
-}
-
-export type VideoTranscodingPayload = HLSTranscodingPayload | NewResolutionTranscodingPayload
-  | OptimizeTranscodingPayload | MergeAudioTranscodingPayload
-
 async function processVideoTranscoding (job: Bull.Job) {
   const payload = job.data as VideoTranscodingPayload
   logger.info('Processing video file in job %d.', job.id)
@@ -105,7 +79,7 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O
 
   const { videoDatabase, videoPublished } = await sequelizeTypescript.transaction(async t => {
     // Maybe the video changed in database, refresh it
-    let videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoArg.uuid, t)
+    const videoDatabase = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoArg.uuid, t)
     // Video does not exist anymore
     if (!videoDatabase) return undefined
 
@@ -118,12 +92,11 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O
 
     let videoPublished = false
 
+    // Generate HLS version of the max quality file
     const hlsPayload = Object.assign({}, payload, { resolution: videoDatabase.getMaxQualityFile().resolution })
     await createHlsJobIfEnabled(hlsPayload)
 
     if (resolutionsEnabled.length !== 0) {
-      const tasks: (Bluebird<Bull.Job<any>> | Promise<Bull.Job<any>>)[] = []
-
       for (const resolution of resolutionsEnabled) {
         let dataInput: VideoTranscodingPayload
 
@@ -143,12 +116,9 @@ async function onVideoFileOptimizerSuccess (videoArg: MVideoWithFile, payload: O
           }
         }
 
-        const p = JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput })
-        tasks.push(p)
+        JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput })
       }
 
-      await Promise.all(tasks)
-
       logger.info('Transcoding jobs created for uuid %s.', videoDatabase.uuid, { resolutionsEnabled })
     } else {
       // No transcoding to do, it's now published
diff --git a/server/lib/job-queue/handlers/video-views.ts b/server/lib/job-queue/handlers/video-views.ts
index 73fa5ed04..7211df237 100644
--- a/server/lib/job-queue/handlers/video-views.ts
+++ b/server/lib/job-queue/handlers/video-views.ts
@@ -3,7 +3,7 @@ import { logger } from '../../../helpers/logger'
 import { VideoModel } from '../../../models/video/video'
 import { VideoViewModel } from '../../../models/video/video-views'
 import { isTestInstance } from '../../../helpers/core-utils'
-import { federateVideoIfNeeded } from '../../activitypub'
+import { federateVideoIfNeeded } from '../../activitypub/videos'
 
 async function processVideosViews () {
   const lastHour = new Date()
@@ -23,6 +23,8 @@ async function processVideosViews () {
   for (const videoId of videoIds) {
     try {
       const views = await Redis.Instance.getVideoViews(videoId, hour)
+      await Redis.Instance.deleteVideoViews(videoId, hour)
+
       if (views) {
         logger.debug('Adding %d views to video %d in hour %d.', views, videoId, hour)
 
@@ -52,8 +54,6 @@ async function processVideosViews () {
           logger.error('Cannot create video views for video %d in hour %d.', videoId, hour, { err })
         }
       }
-
-      await Redis.Instance.deleteVideoViews(videoId, hour)
     } catch (err) {
       logger.error('Cannot update video views of video %d in hour %d.', videoId, hour, { err })
     }