14 files changed, 251 insertions, 131 deletions

diff --git a/server/tests/api/activitypub/client.ts b/server/tests/api/activitypub/client.ts
index d45232c8d..6d90d8643 100644
--- a/server/tests/api/activitypub/client.ts
+++ b/server/tests/api/activitypub/client.ts
@@ -3,32 +3,41 @@
 import * as chai from 'chai'
 import 'mocha'
 import {
+  doubleFollow,
+  flushAndRunMultipleServers,
   flushTests,
   killallServers,
   makeActivityPubGetRequest,
-  runServer,
   ServerInfo,
-  setAccessTokensToServers
+  setAccessTokensToServers,
+  uploadVideo
 } from '../../../../shared/utils'
 
-
 const expect = chai.expect
 
 describe('Test activitypub', function () {
-  let server: ServerInfo = null
+  let servers: ServerInfo[] = []
+  let videoUUID: string
 
   before(async function () {
     this.timeout(30000)
 
     await flushTests()
 
-    server = await runServer(1)
+    servers = await flushAndRunMultipleServers(2)
+
+    await setAccessTokensToServers(servers)
+
+    {
+      const res = await uploadVideo(servers[0].url, servers[0].accessToken, { name: 'video' })
+      videoUUID = res.body.video.uuid
+    }
 
-    await setAccessTokensToServers([ server ])
+    await doubleFollow(servers[0], servers[1])
   })
 
   it('Should return the account object', async function () {
-    const res = await makeActivityPubGetRequest(server.url, '/accounts/root')
+    const res = await makeActivityPubGetRequest(servers[0].url, '/accounts/root')
     const object = res.body
 
     expect(object.type).to.equal('Person')
@@ -37,7 +46,22 @@ describe('Test activitypub', function () {
     expect(object.preferredUsername).to.equal('root')
   })
 
+  it('Should return the video object', async function () {
+    const res = await makeActivityPubGetRequest(servers[0].url, '/videos/watch/' + videoUUID)
+    const object = res.body
+
+    expect(object.type).to.equal('Video')
+    expect(object.id).to.equal('http://localhost:9001/videos/watch/' + videoUUID)
+    expect(object.name).to.equal('video')
+  })
+
+  it('Should redirect to the origin video object', async function () {
+    const res = await makeActivityPubGetRequest(servers[1].url, '/videos/watch/' + videoUUID, 302)
+
+    expect(res.header.location).to.equal('http://localhost:9001/videos/watch/' + videoUUID)
+  })
+
   after(async function () {
-    killallServers([ server ])
+    killallServers(servers)
   })
 })
diff --git a/server/tests/api/activitypub/fetch.ts b/server/tests/api/activitypub/fetch.ts
index e84eb18bb..03609c1a9 100644
--- a/server/tests/api/activitypub/fetch.ts
+++ b/server/tests/api/activitypub/fetch.ts
@@ -11,12 +11,13 @@ import {
   killallServers,
   ServerInfo,
   setAccessTokensToServers,
+  setActorField,
+  setVideoField,
   uploadVideo,
-  userLogin
+  userLogin,
+  waitJobs
 } from '../../../../shared/utils'
 import * as chai from 'chai'
-import { setActorField, setVideoField } from '../../utils/miscs/sql'
-import { waitJobs } from '../../../../shared/utils/server/jobs'
 import { Video } from '../../../../shared/models/videos'
 
 const expect = chai.expect
diff --git a/server/tests/api/activitypub/helpers.ts b/server/tests/api/activitypub/helpers.ts
index 4c42f3d67..ac6e755c3 100644
--- a/server/tests/api/activitypub/helpers.ts
+++ b/server/tests/api/activitypub/helpers.ts
@@ -2,7 +2,7 @@
 
 import 'mocha'
 import { expect } from 'chai'
-import { buildRequestStub } from '../../utils/miscs/stubs'
+import { buildRequestStub } from '../../../../shared/utils/miscs/stubs'
 import { isHTTPSignatureVerified, isJsonLDSignatureVerified, parseHTTPSignature } from '../../../helpers/peertube-crypto'
 import { cloneDeep } from 'lodash'
 import { buildSignedActivity } from '../../../helpers/activitypub'
@@ -91,7 +91,7 @@ describe('Test activity pub helpers', function () {
       req.headers = mastodonObject.headers
       req.headers.signature = 'Signature ' + req.headers.signature
 
-      const parsed = parseHTTPSignature(req, 3600 * 365 * 3)
+      const parsed = parseHTTPSignature(req, 3600 * 1000 * 365 * 10)
       const publicKey = require('./json/mastodon/public-key.json').publicKey
 
       const actor = { publicKey }
@@ -110,7 +110,7 @@ describe('Test activity pub helpers', function () {
       req.headers = mastodonObject.headers
       req.headers.signature = 'Signature ' + req.headers.signature
 
-      const parsed = parseHTTPSignature(req, 3600 * 365 * 3)
+      const parsed = parseHTTPSignature(req, 3600 * 1000 * 365 * 10)
       const publicKey = require('./json/mastodon/bad-public-key.json').publicKey
 
       const actor = { publicKey }
@@ -150,7 +150,7 @@ describe('Test activity pub helpers', function () {
 
       let errored = false
       try {
-        parseHTTPSignature(req, 3600 * 365 * 3)
+        parseHTTPSignature(req, 3600 * 1000 * 365 * 10)
       } catch {
         errored = true
       }
@@ -168,7 +168,7 @@ describe('Test activity pub helpers', function () {
       req.headers = mastodonObject.headers
       req.headers.signature = 'Signature ' + req.headers.signature
 
-      const parsed = parseHTTPSignature(req, 3600 * 365 * 3)
+      const parsed = parseHTTPSignature(req, 3600 * 1000 * 365 * 10)
       const publicKey = require('./json/mastodon/public-key.json').publicKey
 
       const actor = { publicKey }
diff --git a/server/tests/api/activitypub/index.ts b/server/tests/api/activitypub/index.ts
index e748f32e9..450053309 100644
--- a/server/tests/api/activitypub/index.ts
+++ b/server/tests/api/activitypub/index.ts
@@ -1,4 +1,5 @@
 import './client'
 import './fetch'
 import './helpers'
+import './refresher'
 import './security'
diff --git a/server/tests/api/activitypub/refresher.ts b/server/tests/api/activitypub/refresher.ts
new file mode 100644
index 000000000..332ea7ed1
--- /dev/null
+++ b/server/tests/api/activitypub/refresher.ts
@@ -0,0 +1,93 @@
+/* tslint:disable:no-unused-expression */
+
+import 'mocha'
+import {
+  doubleFollow,
+  flushAndRunMultipleServers,
+  getVideo,
+  killallServers,
+  reRunServer,
+  ServerInfo,
+  setAccessTokensToServers,
+  uploadVideo,
+  wait,
+  setVideoField,
+  waitJobs
+} from '../../../../shared/utils'
+
+describe('Test AP refresher', function () {
+  let servers: ServerInfo[] = []
+  let videoUUID1: string
+  let videoUUID2: string
+  let videoUUID3: string
+
+  before(async function () {
+    this.timeout(30000)
+
+    servers = await flushAndRunMultipleServers(2)
+
+    // Get the access tokens
+    await setAccessTokensToServers(servers)
+
+    {
+      const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video1' })
+      videoUUID1 = res.body.video.uuid
+    }
+
+    {
+      const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video2' })
+      videoUUID2 = res.body.video.uuid
+    }
+
+    {
+      const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video3' })
+      videoUUID3 = res.body.video.uuid
+    }
+
+    await doubleFollow(servers[0], servers[1])
+  })
+
+  it('Should remove a deleted remote video', async function () {
+    this.timeout(60000)
+
+    await wait(10000)
+
+    // Change UUID so the remote server returns a 404
+    await setVideoField(2, videoUUID1, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174f')
+
+    await getVideo(servers[0].url, videoUUID1)
+    await getVideo(servers[0].url, videoUUID2)
+
+    await waitJobs(servers)
+
+    await getVideo(servers[0].url, videoUUID1, 404)
+    await getVideo(servers[0].url, videoUUID2, 200)
+  })
+
+  it('Should not update a remote video if the remote instance is down', async function () {
+    this.timeout(60000)
+
+    killallServers([ servers[1] ])
+
+    await setVideoField(2, videoUUID3, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174e')
+
+    // Video will need a refresh
+    await wait(10000)
+
+    await getVideo(servers[0].url, videoUUID3)
+    // The refresh should fail
+    await waitJobs([ servers[0] ])
+
+    await reRunServer(servers[1])
+
+    // Should not refresh the video, even if the last refresh failed (to avoir a loop on dead instances)
+    await getVideo(servers[0].url, videoUUID3)
+    await waitJobs(servers)
+
+    await getVideo(servers[0].url, videoUUID3, 200)
+  })
+
+  after(async function () {
+    killallServers(servers)
+  })
+})
diff --git a/server/tests/api/activitypub/security.ts b/server/tests/api/activitypub/security.ts
index b71a61c8c..342ae0fa1 100644
--- a/server/tests/api/activitypub/security.ts
+++ b/server/tests/api/activitypub/security.ts
@@ -6,14 +6,15 @@ import {
   flushAndRunMultipleServers,
   flushTests,
   killallServers,
-  ServerInfo
+  makeFollowRequest,
+  makePOSTAPRequest,
+  ServerInfo,
+  setActorField
 } from '../../../../shared/utils'
 import { HTTP_SIGNATURE } from '../../../initializers'
 import { buildDigest, buildGlobalHeaders } from '../../../lib/job-queue/handlers/utils/activitypub-http-utils'
 import * as chai from 'chai'
-import { setActorField } from '../../utils/miscs/sql'
 import { activityPubContextify, buildSignedActivity } from '../../../helpers/activitypub'
-import { makeFollowRequest, makePOSTAPRequest } from '../../utils/requests/activitypub'
 
 const expect = chai.expect
 
diff --git a/server/tests/api/check-params/users.ts b/server/tests/api/check-params/users.ts
index f4c177621..05f42bca9 100644
--- a/server/tests/api/check-params/users.ts
+++ b/server/tests/api/check-params/users.ts
@@ -103,13 +103,13 @@ describe('Test users API validators', function () {
     }
 
     it('Should fail with a too small username', async function () {
-      const fields = immutableAssign(baseCorrectParams, { username: 'fi' })
+      const fields = immutableAssign(baseCorrectParams, { username: '' })
 
       await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
     })
 
     it('Should fail with a too long username', async function () {
-      const fields = immutableAssign(baseCorrectParams, { username: 'my_super_username_which_is_very_long' })
+      const fields = immutableAssign(baseCorrectParams, { username: 'super'.repeat(11) })
 
       await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields })
     })
@@ -432,6 +432,14 @@ describe('Test users API validators', function () {
       await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields })
     })
 
+    it('Should fail with an invalid emailVerified attribute', async function () {
+      const fields = {
+        emailVerified: 'yes'
+      }
+
+      await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields })
+    })
+
     it('Should fail with an invalid videoQuota attribute', async function () {
       const fields = {
         videoQuota: -90
@@ -467,6 +475,7 @@ describe('Test users API validators', function () {
     it('Should succeed with the correct params', async function () {
       const fields = {
         email: 'email@example.com',
+        emailVerified: true,
         videoQuota: 42,
         role: UserRole.MODERATOR
       }
@@ -545,13 +554,13 @@ describe('Test users API validators', function () {
     }
 
     it('Should fail with a too small username', async function () {
-      const fields = immutableAssign(baseCorrectParams, { username: 'ji' })
+      const fields = immutableAssign(baseCorrectParams, { username: '' })
 
       await makePostBodyRequest({ url: server.url, path: registrationPath, token: server.accessToken, fields })
     })
 
     it('Should fail with a too long username', async function () {
-      const fields = immutableAssign(baseCorrectParams, { username: 'my_super_username_which_is_very_long' })
+      const fields = immutableAssign(baseCorrectParams, { username: 'super'.repeat(11) })
 
       await makePostBodyRequest({ url: server.url, path: registrationPath, token: server.accessToken, fields })
     })
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index 2bc1b60ce..9d3ce8153 100644
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -137,7 +137,7 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st
   if (!videoUUID) videoUUID = video1Server2UUID
 
   const webseeds = [
-    'http://localhost:9001/static/webseed/' + videoUUID,
+    'http://localhost:9001/static/redundancy/' + videoUUID,
     'http://localhost:9002/static/webseed/' + videoUUID
   ]
 
@@ -149,20 +149,23 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st
     for (const file of video.files) {
       checkMagnetWebseeds(file, webseeds, server)
 
-      // Only servers 1 and 2 have the video
-      if (server.serverNumber !== 3) {
-        await makeGetRequest({
-          url: server.url,
-          statusCodeExpected: 200,
-          path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`,
-          contentType: null
-        })
-      }
+      await makeGetRequest({
+        url: servers[0].url,
+        statusCodeExpected: 200,
+        path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`,
+        contentType: null
+      })
+      await makeGetRequest({
+        url: servers[1].url,
+        statusCodeExpected: 200,
+        path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`,
+        contentType: null
+      })
     }
   }
 
-  for (const directory of [ 'test1', 'test2' ]) {
-    const files = await readdir(join(root(), directory, 'videos'))
+  for (const directory of [ 'test1/redundancy', 'test2/videos' ]) {
+    const files = await readdir(join(root(), directory))
     expect(files).to.have.length.at.least(4)
 
     for (const resolution of [ 240, 360, 480, 720 ]) {
diff --git a/server/tests/api/users/users.ts b/server/tests/api/users/users.ts
index 7dffbb0b1..4914c8ed5 100644
--- a/server/tests/api/users/users.ts
+++ b/server/tests/api/users/users.ts
@@ -478,6 +478,7 @@ describe('Test users', function () {
       userId,
       accessToken,
       email: 'updated2@example.com',
+      emailVerified: true,
       videoQuota: 42,
       role: UserRole.MODERATOR
     })
@@ -487,6 +488,7 @@ describe('Test users', function () {
 
     expect(user.username).to.equal('user_1')
     expect(user.email).to.equal('updated2@example.com')
+    expect(user.emailVerified).to.be.true
     expect(user.nsfwPolicy).to.equal('do_not_list')
     expect(user.videoQuota).to.equal(42)
     expect(user.roleLabel).to.equal('Moderator')
diff --git a/server/tests/cli/index.ts b/server/tests/cli/index.ts
index 6201314ce..c6b7ec078 100644
--- a/server/tests/cli/index.ts
+++ b/server/tests/cli/index.ts
@@ -1,6 +1,7 @@
 // Order of the tests we want to execute
 import './create-import-video-file-job'
 import './create-transcoding-job'
+import './optimize-old-videos'
 import './peertube'
 import './reset-password'
 import './update-host'
diff --git a/server/tests/misc-endpoints.ts b/server/tests/misc-endpoints.ts
index f948fdfd0..5f82719da 100644
--- a/server/tests/misc-endpoints.ts
+++ b/server/tests/misc-endpoints.ts
@@ -2,7 +2,18 @@
 
 import 'mocha'
 import * as chai from 'chai'
-import { flushTests, killallServers, makeGetRequest, runServer, ServerInfo } from '../../shared/utils'
+import {
+  addVideoChannel,
+  createUser,
+  flushTests,
+  killallServers,
+  makeGetRequest,
+  runServer,
+  ServerInfo,
+  setAccessTokensToServers,
+  uploadVideo
+} from '../../shared/utils'
+import { VideoPrivacy } from '../../shared/models/videos'
 
 const expect = chai.expect
 
@@ -15,6 +26,7 @@ describe('Test misc endpoints', function () {
     await flushTests()
 
     server = await runServer(1)
+    await setAccessTokensToServers([ server ])
   })
 
   describe('Test a well known endpoints', function () {
@@ -60,6 +72,16 @@ describe('Test misc endpoints', function () {
 
       expect(res.body.tracking).to.equal('N')
     })
+
+    it('Should get change-password location', async function () {
+      const res = await makeGetRequest({
+        url: server.url,
+        path: '/.well-known/change-password',
+        statusCodeExpected: 302
+      })
+
+      expect(res.header.location).to.equal('/my-account/settings')
+    })
   })
 
   describe('Test classic static endpoints', function () {
@@ -93,6 +115,64 @@ describe('Test misc endpoints', function () {
     })
   })
 
+  describe('Test bots endpoints', function () {
+
+    it('Should get the empty sitemap', async function () {
+      const res = await makeGetRequest({
+        url: server.url,
+        path: '/sitemap.xml',
+        statusCodeExpected: 200
+      })
+
+      expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"')
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/about/instance</loc></url>')
+    })
+
+    it('Should get the empty cached sitemap', async function () {
+      const res = await makeGetRequest({
+        url: server.url,
+        path: '/sitemap.xml',
+        statusCodeExpected: 200
+      })
+
+      expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"')
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/about/instance</loc></url>')
+    })
+
+    it('Should add videos, channel and accounts and get sitemap', async function () {
+      this.timeout(35000)
+
+      await uploadVideo(server.url, server.accessToken, { name: 'video 1', nsfw: false })
+      await uploadVideo(server.url, server.accessToken, { name: 'video 2', nsfw: false })
+      await uploadVideo(server.url, server.accessToken, { name: 'video 3', privacy: VideoPrivacy.PRIVATE })
+
+      await addVideoChannel(server.url, server.accessToken, { name: 'channel1', displayName: 'channel 1' })
+      await addVideoChannel(server.url, server.accessToken, { name: 'channel2', displayName: 'channel 2' })
+
+      await createUser(server.url, server.accessToken, 'user1', 'password')
+      await createUser(server.url, server.accessToken, 'user2', 'password')
+
+      const res = await makeGetRequest({
+        url: server.url,
+        path: '/sitemap.xml?t=1', // avoid using cache
+        statusCodeExpected: 200
+      })
+
+      expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"')
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/about/instance</loc></url>')
+
+      expect(res.text).to.contain('<video:title><![CDATA[video 1]]></video:title>')
+      expect(res.text).to.contain('<video:title><![CDATA[video 2]]></video:title>')
+      expect(res.text).to.not.contain('<video:title><![CDATA[video 3]]></video:title>')
+
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/video-channels/channel1</loc></url>')
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/video-channels/channel2</loc></url>')
+
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/accounts/user1</loc></url>')
+      expect(res.text).to.contain('<url><loc>http://localhost:9001/accounts/user2</loc></url>')
+    })
+  })
+
   after(async function () {
     killallServers([ server ])
   })
diff --git a/server/tests/utils/miscs/sql.ts b/server/tests/utils/miscs/sql.ts
deleted file mode 100644
index 027f78131..000000000
--- a/server/tests/utils/miscs/sql.ts
+++ /dev/null
@@ -1,38 +0,0 @@
-import * as Sequelize from 'sequelize'
-
-function getSequelize (serverNumber: number) {
-  const dbname = 'peertube_test' + serverNumber
-  const username = 'peertube'
-  const password = 'peertube'
-  const host = 'localhost'
-  const port = 5432
-
-  return new Sequelize(dbname, username, password, {
-    dialect: 'postgres',
-    host,
-    port,
-    operatorsAliases: false,
-    logging: false
-  })
-}
-
-function setActorField (serverNumber: number, to: string, field: string, value: string) {
-  const seq = getSequelize(serverNumber)
-
-  const options = { type: Sequelize.QueryTypes.UPDATE }
-
-  return seq.query(`UPDATE actor SET "${field}" = '${value}' WHERE url = '${to}'`, options)
-}
-
-function setVideoField (serverNumber: number, uuid: string, field: string, value: string) {
-  const seq = getSequelize(serverNumber)
-
-  const options = { type: Sequelize.QueryTypes.UPDATE }
-
-  return seq.query(`UPDATE video SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options)
-}
-
-export {
-  setVideoField,
-  setActorField
-}
diff --git a/server/tests/utils/miscs/stubs.ts b/server/tests/utils/miscs/stubs.ts
deleted file mode 100644
index d1eb0e3b2..000000000
--- a/server/tests/utils/miscs/stubs.ts
+++ /dev/null
@@ -1,14 +0,0 @@
-function buildRequestStub (): any {
-  return { }
-}
-
-function buildResponseStub (): any {
-  return {
-    locals: {}
-  }
-}
-
-export {
-  buildResponseStub,
-  buildRequestStub
-}
diff --git a/server/tests/utils/requests/activitypub.ts b/server/tests/utils/requests/activitypub.ts
deleted file mode 100644
index 96fee60a8..000000000
--- a/server/tests/utils/requests/activitypub.ts
+++ /dev/null
@@ -1,43 +0,0 @@
-import { doRequest } from '../../../helpers/requests'
-import { HTTP_SIGNATURE } from '../../../initializers'
-import { buildGlobalHeaders } from '../../../lib/job-queue/handlers/utils/activitypub-http-utils'
-import { activityPubContextify } from '../../../helpers/activitypub'
-
-function makePOSTAPRequest (url: string, body: any, httpSignature: any, headers: any) {
-  const options = {
-    method: 'POST',
-    uri: url,
-    json: body,
-    httpSignature,
-    headers
-  }
-
-  return doRequest(options)
-}
-
-async function makeFollowRequest (to: { url: string }, by: { url: string, privateKey }) {
-  const follow = {
-    type: 'Follow',
-    id: by.url + '/toto',
-    actor: by.url,
-    object: to.url
-  }
-
-  const body = activityPubContextify(follow)
-
-  const httpSignature = {
-    algorithm: HTTP_SIGNATURE.ALGORITHM,
-    authorizationHeaderName: HTTP_SIGNATURE.HEADER_NAME,
-    keyId: by.url,
-    key: by.privateKey,
-    headers: HTTP_SIGNATURE.HEADERS_TO_SIGN
-  }
-  const headers = buildGlobalHeaders(body)
-
-  return makePOSTAPRequest(to.url, body, httpSignature, headers)
-}
-
-export {
-  makePOSTAPRequest,
-  makeFollowRequest
-}