Merge branch 'develop' of https://github.com/Chocobozzz/PeerTube into move-utils-to-shared

19 files changed, 94 insertions, 65 deletions

diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts
index bcbd9be59..79b76fa0b 100644
--- a/server/helpers/activitypub.ts
+++ b/server/helpers/activitypub.ts
@@ -1,7 +1,7 @@
 import * as Bluebird from 'bluebird'
 import * as validator from 'validator'
 import { ResultList } from '../../shared/models'
-import { Activity, ActivityPubActor } from '../../shared/models/activitypub'
+import { Activity } from '../../shared/models/activitypub'
 import { ACTIVITY_PUB } from '../initializers'
 import { ActorModel } from '../models/activitypub/actor'
 import { signJsonLDObject } from './peertube-crypto'
@@ -106,10 +106,10 @@ function buildSignedActivity (byActor: ActorModel, data: Object) {
   return signJsonLDObject(byActor, activity) as Promise<Activity>
 }
 
-function getActorUrl (activityActor: string | ActivityPubActor) {
-  if (typeof activityActor === 'string') return activityActor
+function getAPUrl (activity: string | { id: string }) {
+  if (typeof activity === 'string') return activity
 
-  return activityActor.id
+  return activity.id
 }
 
 function checkUrlsSameHost (url1: string, url2: string) {
@@ -123,7 +123,7 @@ function checkUrlsSameHost (url1: string, url2: string) {
 
 export {
   checkUrlsSameHost,
-  getActorUrl,
+  getAPUrl,
   activityPubContextify,
   activityPubCollectionPagination,
   buildSignedActivity
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts
index 8b9045038..b59e7e40e 100644
--- a/server/helpers/ffmpeg-utils.ts
+++ b/server/helpers/ffmpeg-utils.ts
@@ -1,7 +1,7 @@
 import * as ffmpeg from 'fluent-ffmpeg'
 import { join } from 'path'
 import { getTargetBitrate, VideoResolution } from '../../shared/models/videos'
-import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers'
+import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants'
 import { processImage } from './image-utils'
 import { logger } from './logger'
 import { checkFFmpegEncoders } from '../initializers/checker-before-init'
diff --git a/server/helpers/image-utils.ts b/server/helpers/image-utils.ts
index 3eaa674ed..da3285b13 100644
--- a/server/helpers/image-utils.ts
+++ b/server/helpers/image-utils.ts
@@ -1,13 +1,28 @@
 import 'multer'
 import * as sharp from 'sharp'
-import { remove } from 'fs-extra'
+import { move, remove } from 'fs-extra'
 
 async function processImage (
   physicalFile: { path: string },
   destination: string,
   newSize: { width: number, height: number }
 ) {
-  await sharp(physicalFile.path)
+  if (physicalFile.path === destination) {
+    throw new Error('Sharp needs an input path different that the output path.')
+  }
+
+  const sharpInstance = sharp(physicalFile.path)
+  const metadata = await sharpInstance.metadata()
+
+  // No need to resize
+  if (metadata.width === newSize.width && metadata.height === newSize.height) {
+    await move(physicalFile.path, destination, { overwrite: true })
+    return
+  }
+
+  await remove(destination)
+
+  await sharpInstance
     .resize(newSize.width, newSize.height)
     .toFile(destination)
 
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index dd5b9bf67..40cd659ab 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -119,25 +119,27 @@ export {
 // ---------------------------------------------------------------------------
 
 async function checkPostgresExtensions () {
-  const extensions = [
-    'pg_trgm',
-    'unaccent'
+  const promises = [
+    checkPostgresExtension('pg_trgm'),
+    checkPostgresExtension('unaccent')
   ]
 
-  for (const extension of extensions) {
-    const query = `SELECT true AS enabled FROM pg_available_extensions WHERE name = '${extension}' AND installed_version IS NOT NULL;`
-    const [ res ] = await sequelizeTypescript.query(query, { raw: true })
+  return Promise.all(promises)
+}
+
+async function checkPostgresExtension (extension: string) {
+  const query = `SELECT true AS enabled FROM pg_available_extensions WHERE name = '${extension}' AND installed_version IS NOT NULL;`
+  const [ res ] = await sequelizeTypescript.query(query, { raw: true })
 
-    if (!res || res.length === 0 || res[ 0 ][ 'enabled' ] !== true) {
-      // Try to create the extension ourself
-      try {
-        await sequelizeTypescript.query(`CREATE EXTENSION ${extension};`, { raw: true })
+  if (!res || res.length === 0 || res[ 0 ][ 'enabled' ] !== true) {
+    // Try to create the extension ourself
+    try {
+      await sequelizeTypescript.query(`CREATE EXTENSION ${extension};`, { raw: true })
 
-      } catch {
-        const errorMessage = `You need to enable ${extension} extension in PostgreSQL. ` +
-          `You can do so by running 'CREATE EXTENSION ${extension};' as a PostgreSQL super user in ${CONFIG.DATABASE.DBNAME} database.`
-        throw new Error(errorMessage)
-      }
+    } catch {
+      const errorMessage = `You need to enable ${extension} extension in PostgreSQL. ` +
+        `You can do so by running 'CREATE EXTENSION ${extension};' as a PostgreSQL super user in ${CONFIG.DATABASE.DBNAME} database.`
+      throw new Error(errorMessage)
     }
   }
 }
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts
index c952ad46c..b9a9da183 100644
--- a/server/initializers/installer.ts
+++ b/server/initializers/installer.ts
@@ -12,12 +12,21 @@ import { remove, ensureDir } from 'fs-extra'
 
 async function installApplication () {
   try {
-    await sequelizeTypescript.sync()
-    await removeCacheDirectories()
-    await createDirectoriesIfNotExist()
-    await createApplicationIfNotExist()
-    await createOAuthClientIfNotExist()
-    await createOAuthAdminIfNotExist()
+    await Promise.all([
+      // Database related
+      sequelizeTypescript.sync()
+        .then(() => {
+          return Promise.all([
+            createApplicationIfNotExist(),
+            createOAuthClientIfNotExist(),
+            createOAuthAdminIfNotExist()
+          ])
+        }),
+
+      // Directories
+      removeCacheDirectories()
+        .then(() => createDirectoriesIfNotExist())
+    ])
   } catch (err) {
     logger.error('Cannot install application.', { err })
     process.exit(-1)
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index 218dbc6a7..504263c99 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -5,15 +5,15 @@ import * as url from 'url'
 import * as uuidv4 from 'uuid/v4'
 import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub'
 import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects'
-import { checkUrlsSameHost, getActorUrl } from '../../helpers/activitypub'
+import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub'
 import { isActorObjectValid, normalizeActor } from '../../helpers/custom-validators/activitypub/actor'
 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
 import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
 import { logger } from '../../helpers/logger'
 import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto'
-import { doRequest, doRequestAndSaveToFile, downloadImage } from '../../helpers/requests'
+import { doRequest, downloadImage } from '../../helpers/requests'
 import { getUrlFromWebfinger } from '../../helpers/webfinger'
-import { AVATARS_SIZE, CONFIG, IMAGE_MIMETYPE_EXT, PREVIEWS_SIZE, sequelizeTypescript } from '../../initializers'
+import { AVATARS_SIZE, CONFIG, IMAGE_MIMETYPE_EXT, sequelizeTypescript } from '../../initializers'
 import { AccountModel } from '../../models/account/account'
 import { ActorModel } from '../../models/activitypub/actor'
 import { AvatarModel } from '../../models/avatar/avatar'
@@ -43,7 +43,7 @@ async function getOrCreateActorAndServerAndModel (
   recurseIfNeeded = true,
   updateCollections = false
 ) {
-  const actorUrl = getActorUrl(activityActor)
+  const actorUrl = getAPUrl(activityActor)
   let created = false
 
   let actor = await fetchActorByUrl(actorUrl, fetchType)
diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts
index b9b255ddf..bcc5cac7a 100644
--- a/server/lib/activitypub/process/process.ts
+++ b/server/lib/activitypub/process/process.ts
@@ -1,5 +1,5 @@
 import { Activity, ActivityType } from '../../../../shared/models/activitypub'
-import { checkUrlsSameHost, getActorUrl } from '../../../helpers/activitypub'
+import { checkUrlsSameHost, getAPUrl } from '../../../helpers/activitypub'
 import { logger } from '../../../helpers/logger'
 import { ActorModel } from '../../../models/activitypub/actor'
 import { processAcceptActivity } from './process-accept'
@@ -40,7 +40,7 @@ async function processActivities (
       continue
     }
 
-    const actorUrl = getActorUrl(activity.actor)
+    const actorUrl = getAPUrl(activity.actor)
 
     // When we fetch remote data, we don't have signature
     if (options.signatureActor && actorUrl !== options.signatureActor.url) {
diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts
index d2649e2d5..5dcba778c 100644
--- a/server/lib/activitypub/share.ts
+++ b/server/lib/activitypub/share.ts
@@ -11,7 +11,7 @@ import { doRequest } from '../../helpers/requests'
 import { getOrCreateActorAndServerAndModel } from './actor'
 import { logger } from '../../helpers/logger'
 import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers'
-import { checkUrlsSameHost, getActorUrl } from '../../helpers/activitypub'
+import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub'
 
 async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) {
   if (video.privacy === VideoPrivacy.PRIVATE) return undefined
@@ -41,7 +41,7 @@ async function addVideoShares (shareUrls: string[], instance: VideoModel) {
       })
       if (!body || !body.actor) throw new Error('Body or body actor is invalid')
 
-      const actorUrl = getActorUrl(body.actor)
+      const actorUrl = getAPUrl(body.actor)
       if (checkUrlsSameHost(shareUrl, actorUrl) !== true) {
         throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`)
       }
diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts
index 1854b44c4..2cce67f0c 100644
--- a/server/lib/activitypub/video-rates.ts
+++ b/server/lib/activitypub/video-rates.ts
@@ -9,7 +9,7 @@ import { AccountVideoRateModel } from '../../models/account/account-video-rate'
 import { logger } from '../../helpers/logger'
 import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers'
 import { doRequest } from '../../helpers/requests'
-import { checkUrlsSameHost, getActorUrl } from '../../helpers/activitypub'
+import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub'
 import { ActorModel } from '../../models/activitypub/actor'
 import { getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from './url'
 
@@ -26,7 +26,7 @@ async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRa
       })
       if (!body || !body.actor) throw new Error('Body or body actor is invalid')
 
-      const actorUrl = getActorUrl(body.actor)
+      const actorUrl = getAPUrl(body.actor)
       if (checkUrlsSameHost(actorUrl, rateUrl) !== true) {
         throw new Error(`Rate url ${rateUrl} has not the same host than actor url ${actorUrl}`)
       }
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index 80de92f24..4cecf9345 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -29,7 +29,7 @@ import { createRates } from './video-rates'
 import { addVideoShares, shareVideoByServerAndChannel } from './share'
 import { AccountModel } from '../../models/account/account'
 import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'
-import { checkUrlsSameHost } from '../../helpers/activitypub'
+import { checkUrlsSameHost, getAPUrl } from '../../helpers/activitypub'
 
 async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
   // If the video is not private and published, we federate it
@@ -167,7 +167,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: {
   const refreshViews = options.refreshViews || false
 
   // Get video url
-  const videoUrl = typeof options.videoObject === 'string' ? options.videoObject : options.videoObject.id
+  const videoUrl = getAPUrl(options.videoObject)
 
   let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType)
   if (videoFromDatabase) {
@@ -242,10 +242,6 @@ async function updateVideoFromAP (options: {
       if (options.updateViews === true) options.video.set('views', videoData.views)
       await options.video.save(sequelizeOptions)
 
-      // Don't block on request
-      generateThumbnailFromUrl(options.video, options.videoObject.icon)
-        .catch(err => logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err }))
-
       {
         const videoFileAttributes = videoFileActivityUrlToDBAttributes(options.video, options.videoObject)
         const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a))
@@ -293,6 +289,12 @@ async function updateVideoFromAP (options: {
     logger.debug('Cannot update the remote video.', { err })
     throw err
   }
+
+  try {
+    await generateThumbnailFromUrl(options.video, options.videoObject.icon)
+  } catch (err) {
+    logger.warn('Cannot generate thumbnail of %s.', options.videoObject.id, { err })
+  }
 }
 
 export {
diff --git a/server/lib/user.ts b/server/lib/user.ts
index db29469eb..29d6d087d 100644
--- a/server/lib/user.ts
+++ b/server/lib/user.ts
@@ -18,7 +18,7 @@ async function createUserAccountAndChannel (userToCreate: UserModel, validateUse
     }
 
     const userCreated = await userToCreate.save(userOptions)
-    const accountCreated = await createLocalAccountWithoutKeys(userToCreate.username, userToCreate.id, null, t)
+    const accountCreated = await createLocalAccountWithoutKeys(userCreated.username, userCreated.id, null, t)
     userCreated.Account = accountCreated
 
     let channelName = userCreated.username + '_channel'
@@ -37,8 +37,13 @@ async function createUserAccountAndChannel (userToCreate: UserModel, validateUse
     return { user: userCreated, account: accountCreated, videoChannel }
   })
 
-  account.Actor = await setAsyncActorKeys(account.Actor)
-  videoChannel.Actor = await setAsyncActorKeys(videoChannel.Actor)
+  const [ accountKeys, channelKeys ] = await Promise.all([
+    setAsyncActorKeys(account.Actor),
+    setAsyncActorKeys(videoChannel.Actor)
+  ])
+
+  account.Actor = accountKeys
+  videoChannel.Actor = channelKeys
 
   return { user, account, videoChannel } as { user: UserModel, account: AccountModel, videoChannel: VideoChannelModel }
 }
diff --git a/server/models/account/user.ts b/server/models/account/user.ts
index 34aafa1a7..1843603f1 100644
--- a/server/models/account/user.ts
+++ b/server/models/account/user.ts
@@ -1,6 +1,6 @@
 import * as Sequelize from 'sequelize'
 import {
-  AfterDelete,
+  AfterDestroy,
   AfterUpdate,
   AllowNull,
   BeforeCreate,
@@ -179,7 +179,7 @@ export class UserModel extends Model<UserModel> {
   }
 
   @AfterUpdate
-  @AfterDelete
+  @AfterDestroy
   static removeTokenCache (instance: UserModel) {
     return clearCacheByUserId(instance.id)
   }
diff --git a/server/models/oauth/oauth-token.ts b/server/models/oauth/oauth-token.ts
index ecf846821..08d892da4 100644
--- a/server/models/oauth/oauth-token.ts
+++ b/server/models/oauth/oauth-token.ts
@@ -1,5 +1,5 @@
 import {
-  AfterDelete,
+  AfterDestroy,
   AfterUpdate,
   AllowNull,
   BelongsTo,
@@ -126,7 +126,7 @@ export class OAuthTokenModel extends Model<OAuthTokenModel> {
   OAuthClients: OAuthClientModel[]
 
   @AfterUpdate
-  @AfterDelete
+  @AfterDestroy
   static removeTokenCache (token: OAuthTokenModel) {
     return clearCacheByToken(token.accessToken)
   }
diff --git a/server/tests/api/activitypub/security.ts b/server/tests/api/activitypub/security.ts
index 69b7c0148..b71a61c8c 100644
--- a/server/tests/api/activitypub/security.ts
+++ b/server/tests/api/activitypub/security.ts
@@ -8,15 +8,12 @@ import {
   killallServers,
   ServerInfo
 } from '../../../../shared/utils'
-import {
-  makePOSTAPRequest,
-  makeFollowRequest,
-} from '../../utils/requests/activitypub'
 import { HTTP_SIGNATURE } from '../../../initializers'
 import { buildDigest, buildGlobalHeaders } from '../../../lib/job-queue/handlers/utils/activitypub-http-utils'
 import * as chai from 'chai'
 import { setActorField } from '../../utils/miscs/sql'
 import { activityPubContextify, buildSignedActivity } from '../../../helpers/activitypub'
+import { makeFollowRequest, makePOSTAPRequest } from '../../utils/requests/activitypub'
 
 const expect = chai.expect
 
diff --git a/server/tests/api/users/index.ts b/server/tests/api/users/index.ts
index 0a1b8b0b2..ff433315d 100644
--- a/server/tests/api/users/index.ts
+++ b/server/tests/api/users/index.ts
@@ -1,5 +1,5 @@
 import './blocklist'
 import './user-subscriptions'
 import './users'
-import './users-verification'
 import './users-multiple-servers'
+import './users-verification'
diff --git a/server/tests/api/videos/video-blacklist-management.ts b/server/tests/api/videos/video-blacklist-management.ts
index 06d80a270..61411e30d 100644
--- a/server/tests/api/videos/video-blacklist-management.ts
+++ b/server/tests/api/videos/video-blacklist-management.ts
@@ -1,7 +1,7 @@
 /* tslint:disable:no-unused-expression */
 
 import * as chai from 'chai'
-import * as lodash from 'lodash'
+import { orderBy } from 'lodash'
 import 'mocha'
 import {
   addVideoToBlacklist,
@@ -22,7 +22,6 @@ import { waitJobs } from '../../../../shared/utils/server/jobs'
 import { VideoAbuse } from '../../../../shared/models/videos'
 
 const expect = chai.expect
-const orderBy = lodash.orderBy
 
 describe('Test video blacklist management', function () {
   let servers: ServerInfo[] = []
diff --git a/server/tests/api/videos/video-channels.ts b/server/tests/api/videos/video-channels.ts
index 5bffffc66..63514d69c 100644
--- a/server/tests/api/videos/video-channels.ts
+++ b/server/tests/api/videos/video-channels.ts
@@ -7,10 +7,12 @@ import {
   createUser,
   doubleFollow,
   flushAndRunMultipleServers,
-  getVideoChannelVideos, serverLogin, testImage,
+  getVideoChannelVideos,
+  testImage,
   updateVideo,
   updateVideoChannelAvatar,
-  uploadVideo, wait, userLogin
+  uploadVideo,
+  userLogin
 } from '../../../../shared/utils'
 import {
   addVideoChannel,
diff --git a/server/tests/api/videos/video-schedule-update.ts b/server/tests/api/videos/video-schedule-update.ts
index ecfc5e034..632c4244c 100644
--- a/server/tests/api/videos/video-schedule-update.ts
+++ b/server/tests/api/videos/video-schedule-update.ts
@@ -16,7 +16,6 @@ import {
   uploadVideo,
   wait
 } from '../../../../shared/utils'
-import { join } from 'path'
 import { waitJobs } from '../../../../shared/utils/server/jobs'
 
 const expect = chai.expect
diff --git a/server/tests/api/videos/video-transcoder.ts b/server/tests/api/videos/video-transcoder.ts
index f9458f0ec..68cf00194 100644
--- a/server/tests/api/videos/video-transcoder.ts
+++ b/server/tests/api/videos/video-transcoder.ts
@@ -3,13 +3,13 @@
 import * as chai from 'chai'
 import 'mocha'
 import { omit } from 'lodash'
-import * as ffmpeg from 'fluent-ffmpeg'
 import { getMaxBitrate, VideoDetails, VideoResolution, VideoState } from '../../../../shared/models/videos'
 import { audio, getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
 import {
   buildAbsoluteFixturePath,
   doubleFollow,
   flushAndRunMultipleServers,
+  generateHighBitrateVideo,
   getMyVideos,
   getVideo,
   getVideosList,
@@ -18,8 +18,7 @@ import {
   ServerInfo,
   setAccessTokensToServers,
   uploadVideo,
-  webtorrentAdd,
-  generateHighBitrateVideo
+  webtorrentAdd
 } from '../../../../shared/utils'
 import { join } from 'path'
 import { waitJobs } from '../../../../shared/utils/server/jobs'