Add gitlab ci support

21 files changed, 58 insertions, 36 deletions

diff --git a/server/controllers/api/videos/abuse.ts b/server/controllers/api/videos/abuse.ts
index ca70230a2..77808466c 100644
--- a/server/controllers/api/videos/abuse.ts
+++ b/server/controllers/api/videos/abuse.ts
@@ -113,16 +113,16 @@ async function reportVideoAbuse (req: express.Request, res: express.Response) {
 
     // We send the video abuse to the origin server
     if (videoInstance.isOwned() === false) {
-      await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance)
+      await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance, t)
     }
 
-    Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance)
-
     auditLogger.create(reporterAccount.Actor.getIdentifier(), new VideoAbuseAuditView(videoAbuseInstance.toFormattedJSON()))
 
     return videoAbuseInstance
   })
 
+  Notifier.Instance.notifyOnNewVideoAbuse(videoAbuse)
+
   logger.info('Abuse report for video %s created.', videoInstance.name)
 
   return res.json({ videoAbuse: videoAbuse.toFormattedJSON() }).end()
diff --git a/server/helpers/database-utils.ts b/server/helpers/database-utils.ts
index 39c74b2fd..6c5068fb0 100644
--- a/server/helpers/database-utils.ts
+++ b/server/helpers/database-utils.ts
@@ -2,6 +2,7 @@ import * as retry from 'async/retry'
 import * as Bluebird from 'bluebird'
 import { Model } from 'sequelize-typescript'
 import { logger } from './logger'
+import { Transaction } from 'sequelize'
 
 function retryTransactionWrapper <T, A, B, C> (
   functionToRetry: (arg1: A, arg2: B, arg3: C) => Promise<T> | Bluebird<T>,
@@ -72,11 +73,18 @@ function resetSequelizeInstance (instance: Model<any>, savedFields: object) {
   })
 }
 
+function afterCommitIfTransaction (t: Transaction, fn: Function) {
+  if (t) return t.afterCommit(() => fn())
+
+  return fn()
+}
+
 // ---------------------------------------------------------------------------
 
 export {
   resetSequelizeInstance,
   retryTransactionWrapper,
   transactionRetryer,
-  updateInstanceWithAnother
+  updateInstanceWithAnother,
+  afterCommitIfTransaction
 }
diff --git a/server/lib/activitypub/process/process-flag.ts b/server/lib/activitypub/process/process-flag.ts
index 0b3976089..8faab051e 100644
--- a/server/lib/activitypub/process/process-flag.ts
+++ b/server/lib/activitypub/process/process-flag.ts
@@ -31,7 +31,7 @@ async function processCreateVideoAbuse (activity: ActivityCreate | ActivityFlag,
 
   const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: flag.object })
 
-  return sequelizeTypescript.transaction(async t => {
+  const videoAbuse = await sequelizeTypescript.transaction(async t => {
     const videoAbuseData = {
       reporterAccountId: account.id,
       reason: flag.content,
@@ -42,8 +42,10 @@ async function processCreateVideoAbuse (activity: ActivityCreate | ActivityFlag,
     const videoAbuseInstance = await VideoAbuseModel.create(videoAbuseData, { transaction: t })
     videoAbuseInstance.Video = video
 
-    Notifier.Instance.notifyOnNewVideoAbuse(videoAbuseInstance)
-
     logger.info('Remote abuse for video uuid %s created', flag.object)
+
+    return videoAbuseInstance
   })
+
+  Notifier.Instance.notifyOnNewVideoAbuse(videoAbuse)
 }
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts
index 28f18595b..9c21149f2 100644
--- a/server/lib/activitypub/send/send-create.ts
+++ b/server/lib/activitypub/send/send-create.ts
@@ -11,6 +11,7 @@ import { VideoRedundancyModel } from '../../../models/redundancy/video-redundanc
 import { VideoPlaylistModel } from '../../../models/video/video-playlist'
 import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model'
 import { getServerActor } from '../../../helpers/utils'
+import * as Bluebird from 'bluebird'
 
 async function sendCreateVideo (video: VideoModel, t: Transaction) {
   if (video.privacy === VideoPrivacy.PRIVATE) return undefined
@@ -82,7 +83,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
 
   // This was a reply, send it to the parent actors
   const actorsException = [ byActor ]
-  await broadcastToActors(createActivity, byActor, parentsCommentActors, actorsException)
+  await broadcastToActors(createActivity, byActor, parentsCommentActors, t, actorsException)
 
   // Broadcast to our followers
   await broadcastToFollowers(createActivity, byActor, [ byActor ], t)
@@ -91,7 +92,7 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio
   if (isOrigin) return broadcastToFollowers(createActivity, byActor, actorsInvolvedInComment, t, actorsException)
 
   // Send to origin
-  return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
+  t.afterCommit(() => unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl))
 }
 
 function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate {
diff --git a/server/lib/activitypub/send/send-delete.ts b/server/lib/activitypub/send/send-delete.ts
index 7bf5ca520..7a1d6f0ba 100644
--- a/server/lib/activitypub/send/send-delete.ts
+++ b/server/lib/activitypub/send/send-delete.ts
@@ -59,7 +59,7 @@ async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Trans
 
   // This was a reply, send it to the parent actors
   const actorsException = [ byActor ]
-  await broadcastToActors(activity, byActor, threadParentComments.map(c => c.Account.Actor), actorsException)
+  await broadcastToActors(activity, byActor, threadParentComments.map(c => c.Account.Actor), t, actorsException)
 
   // Broadcast to our followers
   await broadcastToFollowers(activity, byActor, [ byActor ], t)
@@ -68,7 +68,7 @@ async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Trans
   if (isVideoOrigin) return broadcastToFollowers(activity, byActor, actorsInvolvedInComment, t, actorsException)
 
   // Send to origin
-  return unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
+  t.afterCommit(() => unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl))
 }
 
 async function sendDeleteVideoPlaylist (videoPlaylist: VideoPlaylistModel, t: Transaction) {
diff --git a/server/lib/activitypub/send/send-flag.ts b/server/lib/activitypub/send/send-flag.ts
index 96a7311b9..61ee389a6 100644
--- a/server/lib/activitypub/send/send-flag.ts
+++ b/server/lib/activitypub/send/send-flag.ts
@@ -6,8 +6,9 @@ import { unicastTo } from './utils'
 import { logger } from '../../../helpers/logger'
 import { ActivityAudience, ActivityFlag } from '../../../../shared/models/activitypub'
 import { audiencify, getAudience } from '../audience'
+import { Transaction } from 'sequelize'
 
-async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) {
+async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel, t: Transaction) {
   if (!video.VideoChannel.Account.Actor.serverId) return // Local user
 
   const url = getVideoAbuseActivityPubUrl(videoAbuse)
@@ -18,7 +19,7 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel,
   const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] }
   const flagActivity = buildFlagActivity(url, byActor, videoAbuse, audience)
 
-  return unicastTo(flagActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
+  t.afterCommit(() => unicastTo(flagActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl))
 }
 
 function buildFlagActivity (url: string, byActor: ActorModel, videoAbuse: VideoAbuseModel, audience: ActivityAudience): ActivityFlag {
diff --git a/server/lib/activitypub/send/send-follow.ts b/server/lib/activitypub/send/send-follow.ts
index 2c3d02014..c6e7fe83d 100644
--- a/server/lib/activitypub/send/send-follow.ts
+++ b/server/lib/activitypub/send/send-follow.ts
@@ -4,8 +4,9 @@ import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
 import { getActorFollowActivityPubUrl } from '../url'
 import { unicastTo } from './utils'
 import { logger } from '../../../helpers/logger'
+import { Transaction } from 'sequelize'
 
-function sendFollow (actorFollow: ActorFollowModel) {
+function sendFollow (actorFollow: ActorFollowModel, t: Transaction) {
   const me = actorFollow.ActorFollower
   const following = actorFollow.ActorFollowing
 
@@ -17,7 +18,7 @@ function sendFollow (actorFollow: ActorFollowModel) {
   const url = getActorFollowActivityPubUrl(me, following)
   const data = buildFollowActivity(url, me, following)
 
-  return unicastTo(data, me, following.inboxUrl)
+  t.afterCommit(() => unicastTo(data, me, following.inboxUrl))
 }
 
 function buildFollowActivity (url: string, byActor: ActorModel, targetActor: ActorModel): ActivityFollow {
diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts
index 8727a121e..8fcbbac5c 100644
--- a/server/lib/activitypub/send/send-undo.ts
+++ b/server/lib/activitypub/send/send-undo.ts
@@ -37,7 +37,7 @@ async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) {
   const followActivity = buildFollowActivity(followUrl, me, following)
   const undoActivity = undoActivityData(undoUrl, me, followActivity)
 
-  return unicastTo(undoActivity, me, following.inboxUrl)
+  t.afterCommit(() => unicastTo(undoActivity, me, following.inboxUrl))
 }
 
 async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts
index 7411c08d5..5bf092894 100644
--- a/server/lib/activitypub/send/send-update.ts
+++ b/server/lib/activitypub/send/send-update.ts
@@ -26,7 +26,9 @@ async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByAct
   const url = getUpdateActivityPubUrl(video.url, video.updatedAt.toISOString())
 
   // Needed to build the AP object
-  if (!video.VideoCaptions) video.VideoCaptions = await video.$get('VideoCaptions') as VideoCaptionModel[]
+  if (!video.VideoCaptions) {
+    video.VideoCaptions = await video.$get('VideoCaptions', { transaction: t }) as VideoCaptionModel[]
+  }
 
   const videoObject = video.toActivityPubObject()
   const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC)
diff --git a/server/lib/activitypub/send/utils.ts b/server/lib/activitypub/send/utils.ts
index 69706e620..1faae1d84 100644
--- a/server/lib/activitypub/send/utils.ts
+++ b/server/lib/activitypub/send/utils.ts
@@ -7,6 +7,7 @@ import { JobQueue } from '../../job-queue'
 import { VideoModel } from '../../../models/video/video'
 import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience'
 import { getServerActor } from '../../../helpers/utils'
+import { afterCommitIfTransaction } from '../../../helpers/database-utils'
 
 async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: {
   byActor: ActorModel,
@@ -20,7 +21,9 @@ async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAud
     const audience = getRemoteVideoAudience(options.video, actorsInvolvedInVideo)
     const activity = activityBuilder(audience)
 
-    return unicastTo(activity, options.byActor, options.video.VideoChannel.Account.Actor.sharedInboxUrl)
+    return afterCommitIfTransaction(options.transaction, () => {
+      return unicastTo(activity, options.byActor, options.video.VideoChannel.Account.Actor.sharedInboxUrl)
+    })
   }
 
   // Send to followers
@@ -28,6 +31,7 @@ async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAud
   const activity = activityBuilder(audience)
 
   const actorsException = [ options.byActor ]
+
   return broadcastToFollowers(activity, options.byActor, actorsInvolvedInVideo, options.transaction, actorsException)
 }
 
@@ -76,7 +80,7 @@ async function forwardActivity (
     uris,
     body: activity
   }
-  return JobQueue.Instance.createJob({ type: 'activitypub-http-broadcast', payload })
+  return afterCommitIfTransaction(t, () => JobQueue.Instance.createJob({ type: 'activitypub-http-broadcast', payload }))
 }
 
 async function broadcastToFollowers (
@@ -87,20 +91,22 @@ async function broadcastToFollowers (
   actorsException: ActorModel[] = []
 ) {
   const uris = await computeFollowerUris(toFollowersOf, actorsException, t)
-  return broadcastTo(uris, data, byActor)
+
+  return afterCommitIfTransaction(t, () => broadcastTo(uris, data, byActor))
 }
 
 async function broadcastToActors (
   data: any,
   byActor: ActorModel,
   toActors: ActorModel[],
+  t?: Transaction,
   actorsException: ActorModel[] = []
 ) {
   const uris = await computeUris(toActors, actorsException)
-  return broadcastTo(uris, data, byActor)
+  return afterCommitIfTransaction(t, () => broadcastTo(uris, data, byActor))
 }
 
-async function broadcastTo (uris: string[], data: any, byActor: ActorModel) {
+function broadcastTo (uris: string[], data: any, byActor: ActorModel) {
   if (uris.length === 0) return undefined
 
   logger.debug('Creating broadcast job.', { uris })
@@ -114,7 +120,7 @@ async function broadcastTo (uris: string[], data: any, byActor: ActorModel) {
   return JobQueue.Instance.createJob({ type: 'activitypub-http-broadcast', payload })
 }
 
-async function unicastTo (data: any, byActor: ActorModel, toActorUrl: string) {
+function unicastTo (data: any, byActor: ActorModel, toActorUrl: string) {
   logger.debug('Creating unicast job.', { uri: toActorUrl })
 
   const payload = {
@@ -123,7 +129,7 @@ async function unicastTo (data: any, byActor: ActorModel, toActorUrl: string) {
     body: data
   }
 
-  return JobQueue.Instance.createJob({ type: 'activitypub-http-unicast', payload })
+  JobQueue.Instance.createJob({ type: 'activitypub-http-unicast', payload })
 }
 
 // ---------------------------------------------------------------------------
diff --git a/server/lib/job-queue/handlers/activitypub-follow.ts b/server/lib/job-queue/handlers/activitypub-follow.ts
index b3defb617..4ae66cd01 100644
--- a/server/lib/job-queue/handlers/activitypub-follow.ts
+++ b/server/lib/job-queue/handlers/activitypub-follow.ts
@@ -69,7 +69,7 @@ async function follow (fromActor: ActorModel, targetActor: ActorModel) {
     actorFollow.ActorFollower = fromActor
 
     // Send a notification to remote server if our follow is not already accepted
-    if (actorFollow.state !== 'accepted') await sendFollow(actorFollow)
+    if (actorFollow.state !== 'accepted') sendFollow(actorFollow, t)
 
     return actorFollow
   })
diff --git a/server/tests/api/activitypub/fetch.ts b/server/tests/api/activitypub/fetch.ts
index 3a1c0d321..3d54c2042 100644
--- a/server/tests/api/activitypub/fetch.ts
+++ b/server/tests/api/activitypub/fetch.ts
@@ -87,7 +87,7 @@ describe('Test ActivityPub fetcher', function () {
   })
 
   after(async function () {
-    this.timeout(10000)
+    this.timeout(20000)
 
     await cleanupTests(servers)
 
diff --git a/server/tests/api/travis-1.sh b/server/tests/api/ci-1.sh
index db4021b25..8dd9756a5 100644
--- a/server/tests/api/travis-1.sh
+++ b/server/tests/api/ci-1.sh
@@ -6,5 +6,5 @@ checkParamFiles=$(find server/tests/api/check-params -type f | grep -v index.ts
 notificationsFiles=$(find server/tests/api/notifications -type f | grep -v index.ts | xargs echo)
 searchFiles=$(find server/tests/api/search -type f | grep -v index.ts | xargs echo)
 
-MOCHA_PARALLEL=true mocha --timeout 5000 --exit --require ts-node/register --bail \
+MOCHA_PARALLEL=true npm run mocha --timeout 30000 --exit --require ts-node/register --bail \
     $notificationsFiles $searchFiles $checkParamFiles
diff --git a/server/tests/api/travis-2.sh b/server/tests/api/ci-2.sh
index ba7a061b0..16ab585e9 100644
--- a/server/tests/api/travis-2.sh
+++ b/server/tests/api/ci-2.sh
@@ -5,5 +5,5 @@ set -eu
 serverFiles=$(find server/tests/api/server -type f | grep -v index.ts | xargs echo)
 usersFiles=$(find server/tests/api/users -type f | grep -v index.ts | xargs echo)
 
-MOCHA_PARALLEL=true mocha --timeout 5000 --exit --require ts-node/register --bail \
+MOCHA_PARALLEL=true npm run mocha-parallel-tests -- --max-parallel $1 --timeout 30000 --exit --require ts-node/register --bail \
     $serverFiles $usersFiles
diff --git a/server/tests/api/travis-3.sh b/server/tests/api/ci-3.sh
index 82457222c..fc96f6fb4 100644
--- a/server/tests/api/travis-3.sh
+++ b/server/tests/api/ci-3.sh
@@ -4,5 +4,5 @@ set -eu
 
 videosFiles=$(find server/tests/api/videos -type f | grep -v index.ts | xargs echo)
 
-MOCHA_PARALLEL=true mocha --timeout 5000 --exit --require ts-node/register --bail \
+MOCHA_PARALLEL=true npm run mocha --timeout 30000 --exit --require ts-node/register --bail \
     $videosFiles
diff --git a/server/tests/api/travis-4.sh b/server/tests/api/ci-4.sh
index 875986182..74809e1ad 100644
--- a/server/tests/api/travis-4.sh
+++ b/server/tests/api/ci-4.sh
@@ -5,5 +5,5 @@ set -eu
 redundancyFiles=$(find server/tests/api/redundancy -type f | grep -v index.ts | xargs echo)
 activitypubFiles=$(find server/tests/api/activitypub -type f | grep -v index.ts | xargs echo)
 
-MOCHA_PARALLEL=true mocha-parallel-tests --max-parallel $1 --timeout 5000 --exit --require ts-node/register --bail \
+MOCHA_PARALLEL=true npm run mocha-parallel-tests -- --max-parallel $1 --timeout 30000 --exit --require ts-node/register --bail \
     $redundancyFiles $activitypubFiles
diff --git a/server/tests/api/notifications/user-notifications.ts b/server/tests/api/notifications/user-notifications.ts
index 662b64e05..6fa630562 100644
--- a/server/tests/api/notifications/user-notifications.ts
+++ b/server/tests/api/notifications/user-notifications.ts
@@ -782,7 +782,7 @@ describe('Test users notifications', function () {
     it('Should not send a notification before the video is published', async function () {
       this.timeout(20000)
 
-      let updateAt = new Date(new Date().getTime() + 100000)
+      let updateAt = new Date(new Date().getTime() + 1000000)
 
       const data = {
         privacy: VideoPrivacy.PRIVATE,
@@ -1074,7 +1074,7 @@ describe('Test users notifications', function () {
     it('Should send unblacklist but not published/subscription notes after unblacklisted if scheduled update pending', async function () {
       this.timeout(20000)
 
-      let updateAt = new Date(new Date().getTime() + 100000)
+      let updateAt = new Date(new Date().getTime() + 1000000)
 
       const name = 'video with auto-blacklist and future schedule ' + uuidv4()
 
diff --git a/server/tests/api/server/email.ts b/server/tests/api/server/email.ts
index 7b7acfd12..c55a221f2 100644
--- a/server/tests/api/server/email.ts
+++ b/server/tests/api/server/email.ts
@@ -40,7 +40,7 @@ describe('Test emails', function () {
   let emailPort: number
 
   before(async function () {
-    this.timeout(30000)
+    this.timeout(50000)
 
     emailPort = await MockSmtpServer.Instance.collectEmails(emails)
 
diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts
index 068654d8c..a225443c5 100644
--- a/server/tests/api/server/handle-down.ts
+++ b/server/tests/api/server/handle-down.ts
@@ -109,7 +109,7 @@ describe('Test handle downs', function () {
   })
 
   it('Should remove followers that are often down', async function () {
-    this.timeout(60000)
+    this.timeout(240000)
 
     // Server 2 and 3 follow server 1
     await follow(servers[1].url, [ servers[0].url ], servers[1].accessToken)
diff --git a/server/tests/api/server/jobs.ts b/server/tests/api/server/jobs.ts
index 3ab2fe120..ceea47a85 100644
--- a/server/tests/api/server/jobs.ts
+++ b/server/tests/api/server/jobs.ts
@@ -8,6 +8,7 @@ import { getJobsList, getJobsListPaginationAndSort, waitJobs } from '../../../..
 import { flushAndRunMultipleServers } from '../../../../shared/extra-utils/server/servers'
 import { uploadVideo } from '../../../../shared/extra-utils/videos/videos'
 import { dateIsValid } from '../../../../shared/extra-utils/miscs/miscs'
+import { Job } from '../../../../shared/models/server'
 
 const expect = chai.expect
 
@@ -50,7 +51,7 @@ describe('Test jobs', function () {
     if (job.type === 'videos-views') job = res.body.data[1]
 
     expect(job.state).to.equal('completed')
-    expect(job.type).to.equal('activitypub-follow')
+    expect(job.type.startsWith('activitypub-')).to.be.true
     expect(dateIsValid(job.createdAt)).to.be.true
     expect(dateIsValid(job.processedOn)).to.be.true
     expect(dateIsValid(job.finishedOn)).to.be.true
diff --git a/server/tests/api/videos/video-schedule-update.ts b/server/tests/api/videos/video-schedule-update.ts
index 64f657780..65a8eafb8 100644
--- a/server/tests/api/videos/video-schedule-update.ts
+++ b/server/tests/api/videos/video-schedule-update.ts
@@ -85,7 +85,7 @@ describe('Test video update scheduler', function () {
   })
 
   it('Should wait some seconds and have the video in public privacy', async function () {
-    this.timeout(20000)
+    this.timeout(50000)
 
     await wait(15000)
     await waitJobs(servers)