Basic video redundancy implementation

5 files changed, 249 insertions, 12 deletions

diff --git a/server/tests/api/check-params/follows.ts b/server/tests/api/check-params/follows.ts
index 2bc3b27d9..cdc95c81a 100644
--- a/server/tests/api/check-params/follows.ts
+++ b/server/tests/api/check-params/follows.ts
@@ -169,15 +169,6 @@ describe('Test server follows API validators', function () {
           statusCodeExpected: 404
         })
       })
-
-      it('Should succeed with the correct parameters', async function () {
-        await makeDeleteRequest({
-          url: server.url,
-          path: path + '/localhost:9002',
-          token: server.accessToken,
-          statusCodeExpected: 404
-        })
-      })
     })
   })
 
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts
index 777acbb0f..44460a167 100644
--- a/server/tests/api/check-params/index.ts
+++ b/server/tests/api/check-params/index.ts
@@ -1,15 +1,17 @@
 // Order of the tests we want to execute
 import './accounts'
+import './config'
 import './follows'
 import './jobs'
+import './redundancy'
+import './search'
 import './services'
+import './user-subscriptions'
 import './users'
 import './video-abuses'
 import './video-blacklist'
 import './video-captions'
 import './video-channels'
 import './video-comments'
-import './videos'
 import './video-imports'
-import './search'
-import './user-subscriptions'
+import './videos'
diff --git a/server/tests/api/check-params/redundancy.ts b/server/tests/api/check-params/redundancy.ts
new file mode 100644
index 000000000..aa588e3dd
--- /dev/null
+++ b/server/tests/api/check-params/redundancy.ts
@@ -0,0 +1,103 @@
+/* tslint:disable:no-unused-expression */
+
+import 'mocha'
+
+import {
+  createUser,
+  doubleFollow,
+  flushAndRunMultipleServers,
+  flushTests,
+  killallServers,
+  makePutBodyRequest,
+  ServerInfo,
+  setAccessTokensToServers,
+  userLogin
+} from '../../utils'
+
+describe('Test server redundancy API validators', function () {
+  let servers: ServerInfo[]
+  let userAccessToken = null
+
+  // ---------------------------------------------------------------
+
+  before(async function () {
+    this.timeout(30000)
+
+    await flushTests()
+    servers = await flushAndRunMultipleServers(2)
+
+    await setAccessTokensToServers(servers)
+    await doubleFollow(servers[0], servers[1])
+
+    const user = {
+      username: 'user1',
+      password: 'password'
+    }
+
+    await createUser(servers[0].url, servers[0].accessToken, user.username, user.password)
+    userAccessToken = await userLogin(servers[0], user)
+  })
+
+  describe('When updating redundancy', function () {
+    const path = '/api/v1/server/redundancy'
+
+    it('Should fail with an invalid token', async function () {
+      await makePutBodyRequest({
+        url: servers[0].url,
+        path: path + '/localhost:9002',
+        fields: { redundancyAllowed: true },
+        token: 'fake_token',
+        statusCodeExpected: 401
+      })
+    })
+
+    it('Should fail if the user is not an administrator', async function () {
+      await makePutBodyRequest({
+        url: servers[0].url,
+        path: path + '/localhost:9002',
+        fields: { redundancyAllowed: true },
+        token: userAccessToken,
+        statusCodeExpected: 403
+      })
+    })
+
+    it('Should fail if we do not follow this server', async function () {
+      await makePutBodyRequest({
+        url: servers[0].url,
+        path: path + '/example.com',
+        fields: { redundancyAllowed: true },
+        token: servers[0].accessToken,
+        statusCodeExpected: 404
+      })
+    })
+
+    it('Should fail without de redundancyAllowed param', async function () {
+      await makePutBodyRequest({
+        url: servers[0].url,
+        path: path + '/localhost:9002',
+        fields: { blabla: true },
+        token: servers[0].accessToken,
+        statusCodeExpected: 400
+      })
+    })
+
+    it('Should succeed with the correct parameters', async function () {
+      await makePutBodyRequest({
+        url: servers[0].url,
+        path: path + '/localhost:9002',
+        fields: { redundancyAllowed: true },
+        token: servers[0].accessToken,
+        statusCodeExpected: 204
+      })
+    })
+  })
+
+  after(async function () {
+    killallServers(servers)
+
+    // Keep the logs if the test failed
+    if (this['ok']) {
+      await flushTests()
+    }
+  })
+})
diff --git a/server/tests/api/server/index.ts b/server/tests/api/server/index.ts
index eeb8b7a28..c74c68a33 100644
--- a/server/tests/api/server/index.ts
+++ b/server/tests/api/server/index.ts
@@ -3,6 +3,7 @@ import './email'
 import './follows'
 import './handle-down'
 import './jobs'
+import './redundancy'
 import './reverse-proxy'
 import './stats'
 import './tracker'
diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts
new file mode 100644
index 000000000..c0ec75a45
--- /dev/null
+++ b/server/tests/api/server/redundancy.ts
@@ -0,0 +1,140 @@
+/* tslint:disable:no-unused-expression */
+
+import * as chai from 'chai'
+import 'mocha'
+import { VideoDetails } from '../../../../shared/models/videos'
+import {
+  doubleFollow,
+  flushAndRunMultipleServers,
+  flushTests,
+  getFollowingListPaginationAndSort,
+  getVideo,
+  killallServers,
+  ServerInfo,
+  setAccessTokensToServers,
+  uploadVideo,
+  wait,
+  root, viewVideo
+} from '../../utils'
+import { waitJobs } from '../../utils/server/jobs'
+import * as magnetUtil from 'magnet-uri'
+import { updateRedundancy } from '../../utils/server/redundancy'
+import { ActorFollow } from '../../../../shared/models/actors'
+import { readdir } from 'fs-extra'
+import { join } from 'path'
+
+const expect = chai.expect
+
+function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) {
+  const parsed = magnetUtil.decode(file.magnetUri)
+
+  for (const ws of baseWebseeds) {
+    const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
+    expect(found, `Webseed ${ws} not found in ${file.magnetUri}`).to.not.be.undefined
+  }
+}
+
+describe('Test videos redundancy', function () {
+  let servers: ServerInfo[] = []
+  let video1Server2UUID: string
+  let video2Server2UUID: string
+
+  before(async function () {
+    this.timeout(120000)
+
+    servers = await flushAndRunMultipleServers(3)
+
+    // Get the access tokens
+    await setAccessTokensToServers(servers)
+
+    {
+      const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
+      video1Server2UUID = res.body.video.uuid
+
+      await viewVideo(servers[1].url, video1Server2UUID)
+    }
+
+    {
+      const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
+      video2Server2UUID = res.body.video.uuid
+    }
+
+    await waitJobs(servers)
+
+    // Server 1 and server 2 follow each other
+    await doubleFollow(servers[0], servers[1])
+    // Server 1 and server 3 follow each other
+    await doubleFollow(servers[0], servers[2])
+    // Server 2 and server 3 follow each other
+    await doubleFollow(servers[1], servers[2])
+
+    await waitJobs(servers)
+  })
+
+  it('Should have 1 webseed on the first video', async function () {
+    const webseeds = [
+      'http://localhost:9002/static/webseed/' + video1Server2UUID
+    ]
+
+    for (const server of servers) {
+      const res = await getVideo(server.url, video1Server2UUID)
+
+      const video: VideoDetails = res.body
+      video.files.forEach(f => checkMagnetWebseeds(f, webseeds))
+    }
+  })
+
+  it('Should enable redundancy on server 1', async function () {
+    await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true)
+
+    const res = await getFollowingListPaginationAndSort(servers[0].url, 0, 5, '-createdAt')
+    const follows: ActorFollow[] = res.body.data
+    const server2 = follows.find(f => f.following.host === 'localhost:9002')
+    const server3 = follows.find(f => f.following.host === 'localhost:9003')
+
+    expect(server3).to.not.be.undefined
+    expect(server3.following.hostRedundancyAllowed).to.be.false
+
+    expect(server2).to.not.be.undefined
+    expect(server2.following.hostRedundancyAllowed).to.be.true
+  })
+
+  it('Should have 2 webseed on the first video', async function () {
+    this.timeout(40000)
+
+    await waitJobs(servers)
+    await wait(15000)
+    await waitJobs(servers)
+
+    const webseeds = [
+      'http://localhost:9001/static/webseed/' + video1Server2UUID,
+      'http://localhost:9002/static/webseed/' + video1Server2UUID
+    ]
+
+    for (const server of servers) {
+      const res = await getVideo(server.url, video1Server2UUID)
+
+      const video: VideoDetails = res.body
+
+      for (const file of video.files) {
+        checkMagnetWebseeds(file, webseeds)
+      }
+    }
+
+    const files = await readdir(join(root(), 'test1', 'videos'))
+    expect(files).to.have.lengthOf(4)
+
+    for (const resolution of [ 240, 360, 480, 720 ]) {
+      expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined
+    }
+  })
+
+  after(async function () {
+    killallServers(servers)
+
+    // Keep the logs if the test failed
+    if (this['ok']) {
+      await flushTests()
+    }
+  })
+})