Merge branch 'develop' into pr/1217

9 files changed, 618 insertions, 0 deletions

diff --git a/shared/utils/server/activitypub.ts b/shared/utils/server/activitypub.ts
new file mode 100644
index 000000000..eccb198ca
--- /dev/null
+++ b/shared/utils/server/activitypub.ts
@@ -0,0 +1,14 @@
+import * as request from 'supertest'
+
+function makeActivityPubGetRequest (url: string, path: string, expectedStatus = 200) {
+  return request(url)
+    .get(path)
+    .set('Accept', 'application/activity+json,text/html;q=0.9,\\*/\\*;q=0.8')
+    .expect(expectedStatus)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  makeActivityPubGetRequest
+}
diff --git a/shared/utils/server/clients.ts b/shared/utils/server/clients.ts
new file mode 100644
index 000000000..273aac747
--- /dev/null
+++ b/shared/utils/server/clients.ts
@@ -0,0 +1,19 @@
+import * as request from 'supertest'
+import * as urlUtil from 'url'
+
+function getClient (url: string) {
+  const path = '/api/v1/oauth-clients/local'
+
+  return request(url)
+          .get(path)
+          .set('Host', urlUtil.parse(url).host)
+          .set('Accept', 'application/json')
+          .expect(200)
+          .expect('Content-Type', /json/)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  getClient
+}
diff --git a/shared/utils/server/config.ts b/shared/utils/server/config.ts
new file mode 100644
index 000000000..29c24cff9
--- /dev/null
+++ b/shared/utils/server/config.ts
@@ -0,0 +1,142 @@
+import { makeDeleteRequest, makeGetRequest, makePutBodyRequest } from '../requests/requests'
+import { CustomConfig } from '../../models/server/custom-config.model'
+
+function getConfig (url: string) {
+  const path = '/api/v1/config'
+
+  return makeGetRequest({
+    url,
+    path,
+    statusCodeExpected: 200
+  })
+}
+
+function getAbout (url: string) {
+  const path = '/api/v1/config/about'
+
+  return makeGetRequest({
+    url,
+    path,
+    statusCodeExpected: 200
+  })
+}
+
+function getCustomConfig (url: string, token: string, statusCodeExpected = 200) {
+  const path = '/api/v1/config/custom'
+
+  return makeGetRequest({
+    url,
+    token,
+    path,
+    statusCodeExpected
+  })
+}
+
+function updateCustomConfig (url: string, token: string, newCustomConfig: CustomConfig, statusCodeExpected = 200) {
+  const path = '/api/v1/config/custom'
+
+  return makePutBodyRequest({
+    url,
+    token,
+    path,
+    fields: newCustomConfig,
+    statusCodeExpected
+  })
+}
+
+function updateCustomSubConfig (url: string, token: string, newConfig: any) {
+  const updateParams: CustomConfig = {
+    instance: {
+      name: 'PeerTube updated',
+      shortDescription: 'my short description',
+      description: 'my super description',
+      terms: 'my super terms',
+      defaultClientRoute: '/videos/recently-added',
+      defaultNSFWPolicy: 'blur',
+      customizations: {
+        javascript: 'alert("coucou")',
+        css: 'body { background-color: red; }'
+      }
+    },
+    services: {
+      twitter: {
+        username: '@MySuperUsername',
+        whitelisted: true
+      }
+    },
+    cache: {
+      previews: {
+        size: 2
+      },
+      captions: {
+        size: 3
+      }
+    },
+    signup: {
+      enabled: false,
+      limit: 5,
+      requiresEmailVerification: false
+    },
+    admin: {
+      email: 'superadmin1@example.com'
+    },
+    contactForm: {
+      enabled: true
+    },
+    user: {
+      videoQuota: 5242881,
+      videoQuotaDaily: 318742
+    },
+    transcoding: {
+      enabled: true,
+      allowAdditionalExtensions: true,
+      threads: 1,
+      resolutions: {
+        '240p': false,
+        '360p': true,
+        '480p': true,
+        '720p': false,
+        '1080p': false
+      },
+      hls: {
+        enabled: false
+      }
+    },
+    import: {
+      videos: {
+        http: {
+          enabled: false
+        },
+        torrent: {
+          enabled: false
+        }
+      }
+    }
+  }
+
+  Object.assign(updateParams, newConfig)
+
+  return updateCustomConfig(url, token, updateParams)
+}
+
+function deleteCustomConfig (url: string, token: string, statusCodeExpected = 200) {
+  const path = '/api/v1/config/custom'
+
+  return makeDeleteRequest({
+    url,
+    token,
+    path,
+    statusCodeExpected
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  getConfig,
+  getCustomConfig,
+  updateCustomConfig,
+  getAbout,
+  deleteCustomConfig,
+  updateCustomSubConfig
+}
diff --git a/shared/utils/server/contact-form.ts b/shared/utils/server/contact-form.ts
new file mode 100644
index 000000000..80394cf99
--- /dev/null
+++ b/shared/utils/server/contact-form.ts
@@ -0,0 +1,28 @@
+import * as request from 'supertest'
+import { ContactForm } from '../../models/server'
+
+function sendContactForm (options: {
+  url: string,
+  fromEmail: string,
+  fromName: string,
+  body: string,
+  expectedStatus?: number
+}) {
+  const path = '/api/v1/server/contact'
+
+  const body: ContactForm = {
+    fromEmail: options.fromEmail,
+    fromName: options.fromName,
+    body: options.body
+  }
+  return request(options.url)
+    .post(path)
+    .send(body)
+    .expect(options.expectedStatus || 204)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  sendContactForm
+}
diff --git a/shared/utils/server/follows.ts b/shared/utils/server/follows.ts
new file mode 100644
index 000000000..7741757a6
--- /dev/null
+++ b/shared/utils/server/follows.ts
@@ -0,0 +1,79 @@
+import * as request from 'supertest'
+import { ServerInfo } from './servers'
+import { waitJobs } from './jobs'
+
+function getFollowersListPaginationAndSort (url: string, start: number, count: number, sort: string, search?: string) {
+  const path = '/api/v1/server/followers'
+
+  return request(url)
+    .get(path)
+    .query({ start })
+    .query({ count })
+    .query({ sort })
+    .query({ search })
+    .set('Accept', 'application/json')
+    .expect(200)
+    .expect('Content-Type', /json/)
+}
+
+function getFollowingListPaginationAndSort (url: string, start: number, count: number, sort: string, search?: string) {
+  const path = '/api/v1/server/following'
+
+  return request(url)
+    .get(path)
+    .query({ start })
+    .query({ count })
+    .query({ sort })
+    .query({ search })
+    .set('Accept', 'application/json')
+    .expect(200)
+    .expect('Content-Type', /json/)
+}
+
+async function follow (follower: string, following: string[], accessToken: string, expectedStatus = 204) {
+  const path = '/api/v1/server/following'
+
+  const followingHosts = following.map(f => f.replace(/^http:\/\//, ''))
+  const res = await request(follower)
+    .post(path)
+    .set('Accept', 'application/json')
+    .set('Authorization', 'Bearer ' + accessToken)
+    .send({ 'hosts': followingHosts })
+    .expect(expectedStatus)
+
+  return res
+}
+
+async function unfollow (url: string, accessToken: string, target: ServerInfo, expectedStatus = 204) {
+  const path = '/api/v1/server/following/' + target.host
+
+  const res = await request(url)
+    .delete(path)
+    .set('Accept', 'application/json')
+    .set('Authorization', 'Bearer ' + accessToken)
+    .expect(expectedStatus)
+
+  return res
+}
+
+async function doubleFollow (server1: ServerInfo, server2: ServerInfo) {
+  await Promise.all([
+    follow(server1.url, [ server2.url ], server1.accessToken),
+    follow(server2.url, [ server1.url ], server2.accessToken)
+  ])
+
+  // Wait request propagation
+  await waitJobs([ server1, server2 ])
+
+  return true
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  getFollowersListPaginationAndSort,
+  getFollowingListPaginationAndSort,
+  unfollow,
+  follow,
+  doubleFollow
+}
diff --git a/shared/utils/server/jobs.ts b/shared/utils/server/jobs.ts
new file mode 100644
index 000000000..692b5e24d
--- /dev/null
+++ b/shared/utils/server/jobs.ts
@@ -0,0 +1,82 @@
+import * as request from 'supertest'
+import { Job, JobState } from '../../models'
+import { wait } from '../miscs/miscs'
+import { ServerInfo } from './servers'
+
+function getJobsList (url: string, accessToken: string, state: JobState) {
+  const path = '/api/v1/jobs/' + state
+
+  return request(url)
+          .get(path)
+          .set('Accept', 'application/json')
+          .set('Authorization', 'Bearer ' + accessToken)
+          .expect(200)
+          .expect('Content-Type', /json/)
+}
+
+function getJobsListPaginationAndSort (url: string, accessToken: string, state: JobState, start: number, count: number, sort: string) {
+  const path = '/api/v1/jobs/' + state
+
+  return request(url)
+          .get(path)
+          .query({ start })
+          .query({ count })
+          .query({ sort })
+          .set('Accept', 'application/json')
+          .set('Authorization', 'Bearer ' + accessToken)
+          .expect(200)
+          .expect('Content-Type', /json/)
+}
+
+async function waitJobs (serversArg: ServerInfo[] | ServerInfo) {
+  const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) : 2000
+  let servers: ServerInfo[]
+
+  if (Array.isArray(serversArg) === false) servers = [ serversArg as ServerInfo ]
+  else servers = serversArg as ServerInfo[]
+
+  const states: JobState[] = [ 'waiting', 'active', 'delayed' ]
+  let pendingRequests = false
+
+  function tasksBuilder () {
+    const tasks: Promise<any>[] = []
+    pendingRequests = false
+
+    // Check if each server has pending request
+    for (const server of servers) {
+      for (const state of states) {
+        const p = getJobsListPaginationAndSort(server.url, server.accessToken, state, 0, 10, '-createdAt')
+          .then(res => res.body.data)
+          .then((jobs: Job[]) => jobs.filter(j => j.type !== 'videos-views'))
+          .then(jobs => {
+            if (jobs.length !== 0) pendingRequests = true
+          })
+        tasks.push(p)
+      }
+    }
+
+    return tasks
+  }
+
+  do {
+    await Promise.all(tasksBuilder())
+
+    // Retry, in case of new jobs were created
+    if (pendingRequests === false) {
+      await wait(pendingJobWait)
+      await Promise.all(tasksBuilder())
+    }
+
+    if (pendingRequests) {
+      await wait(1000)
+    }
+  } while (pendingRequests)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  getJobsList,
+  waitJobs,
+  getJobsListPaginationAndSort
+}
diff --git a/shared/utils/server/redundancy.ts b/shared/utils/server/redundancy.ts
new file mode 100644
index 000000000..c39ff2c8b
--- /dev/null
+++ b/shared/utils/server/redundancy.ts
@@ -0,0 +1,17 @@
+import { makePutBodyRequest } from '../requests/requests'
+
+async function updateRedundancy (url: string, accessToken: string, host: string, redundancyAllowed: boolean, expectedStatus = 204) {
+  const path = '/api/v1/server/redundancy/' + host
+
+  return makePutBodyRequest({
+    url,
+    path,
+    token: accessToken,
+    fields: { redundancyAllowed },
+    statusCodeExpected: expectedStatus
+  })
+}
+
+export {
+  updateRedundancy
+}
diff --git a/shared/utils/server/servers.ts b/shared/utils/server/servers.ts
new file mode 100644
index 000000000..bde7dd5c2
--- /dev/null
+++ b/shared/utils/server/servers.ts
@@ -0,0 +1,215 @@
+/* tslint:disable:no-unused-expression */
+
+import { ChildProcess, exec, fork } from 'child_process'
+import { join } from 'path'
+import { root, wait } from '../miscs/miscs'
+import { readdir, readFile } from 'fs-extra'
+import { existsSync } from 'fs'
+import { expect } from 'chai'
+
+interface ServerInfo {
+  app: ChildProcess,
+  url: string
+  host: string
+  serverNumber: number
+
+  client: {
+    id: string,
+    secret: string
+  }
+
+  user: {
+    username: string,
+    password: string,
+    email?: string
+  }
+
+  accessToken?: string
+
+  video?: {
+    id: number
+    uuid: string
+    name: string
+    account: {
+      name: string
+    }
+  }
+
+  remoteVideo?: {
+    id: number
+    uuid: string
+  }
+}
+
+function flushAndRunMultipleServers (totalServers: number, configOverride?: Object) {
+  let apps = []
+  let i = 0
+
+  return new Promise<ServerInfo[]>(res => {
+    function anotherServerDone (serverNumber, app) {
+      apps[serverNumber - 1] = app
+      i++
+      if (i === totalServers) {
+        return res(apps)
+      }
+    }
+
+    flushTests()
+      .then(() => {
+        for (let j = 1; j <= totalServers; j++) {
+          runServer(j, configOverride).then(app => anotherServerDone(j, app))
+        }
+      })
+  })
+}
+
+function flushTests () {
+  return new Promise<void>((res, rej) => {
+    return exec('npm run clean:server:test', err => {
+      if (err) return rej(err)
+
+      return res()
+    })
+  })
+}
+
+function runServer (serverNumber: number, configOverride?: Object, args = []) {
+  const server: ServerInfo = {
+    app: null,
+    serverNumber: serverNumber,
+    url: `http://localhost:${9000 + serverNumber}`,
+    host: `localhost:${9000 + serverNumber}`,
+    client: {
+      id: null,
+      secret: null
+    },
+    user: {
+      username: null,
+      password: null
+    }
+  }
+
+  // These actions are async so we need to be sure that they have both been done
+  const serverRunString = {
+    'Server listening': false
+  }
+  const key = 'Database peertube_test' + serverNumber + ' is ready'
+  serverRunString[key] = false
+
+  const regexps = {
+    client_id: 'Client id: (.+)',
+    client_secret: 'Client secret: (.+)',
+    user_username: 'Username: (.+)',
+    user_password: 'User password: (.+)'
+  }
+
+  // Share the environment
+  const env = Object.create(process.env)
+  env['NODE_ENV'] = 'test'
+  env['NODE_APP_INSTANCE'] = serverNumber.toString()
+
+  if (configOverride !== undefined) {
+    env['NODE_CONFIG'] = JSON.stringify(configOverride)
+  }
+
+  const options = {
+    silent: true,
+    env: env,
+    detached: true
+  }
+
+  return new Promise<ServerInfo>(res => {
+    server.app = fork(join(root(), 'dist', 'server.js'), args, options)
+    server.app.stdout.on('data', function onStdout (data) {
+      let dontContinue = false
+
+      // Capture things if we want to
+      for (const key of Object.keys(regexps)) {
+        const regexp = regexps[key]
+        const matches = data.toString().match(regexp)
+        if (matches !== null) {
+          if (key === 'client_id') server.client.id = matches[1]
+          else if (key === 'client_secret') server.client.secret = matches[1]
+          else if (key === 'user_username') server.user.username = matches[1]
+          else if (key === 'user_password') server.user.password = matches[1]
+        }
+      }
+
+      // Check if all required sentences are here
+      for (const key of Object.keys(serverRunString)) {
+        if (data.toString().indexOf(key) !== -1) serverRunString[key] = true
+        if (serverRunString[key] === false) dontContinue = true
+      }
+
+      // If no, there is maybe one thing not already initialized (client/user credentials generation...)
+      if (dontContinue === true) return
+
+      server.app.stdout.removeListener('data', onStdout)
+
+      process.on('exit', () => {
+        try {
+          process.kill(server.app.pid)
+        } catch { /* empty */ }
+      })
+
+      res(server)
+    })
+
+  })
+}
+
+async function reRunServer (server: ServerInfo, configOverride?: any) {
+  const newServer = await runServer(server.serverNumber, configOverride)
+  server.app = newServer.app
+
+  return server
+}
+
+async function checkTmpIsEmpty (server: ServerInfo) {
+  return checkDirectoryIsEmpty(server, 'tmp')
+}
+
+async function checkDirectoryIsEmpty (server: ServerInfo, directory: string) {
+  const testDirectory = 'test' + server.serverNumber
+
+  const directoryPath = join(root(), testDirectory, directory)
+
+  const directoryExists = existsSync(directoryPath)
+  expect(directoryExists).to.be.true
+
+  const files = await readdir(directoryPath)
+  expect(files).to.have.lengthOf(0)
+}
+
+function killallServers (servers: ServerInfo[]) {
+  for (const server of servers) {
+    process.kill(-server.app.pid)
+  }
+}
+
+async function waitUntilLog (server: ServerInfo, str: string, count = 1) {
+  const logfile = join(root(), 'test' + server.serverNumber, 'logs/peertube.log')
+
+  while (true) {
+    const buf = await readFile(logfile)
+
+    const matches = buf.toString().match(new RegExp(str, 'g'))
+    if (matches && matches.length === count) return
+
+    await wait(1000)
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  checkDirectoryIsEmpty,
+  checkTmpIsEmpty,
+  ServerInfo,
+  flushAndRunMultipleServers,
+  flushTests,
+  runServer,
+  killallServers,
+  reRunServer,
+  waitUntilLog
+}
diff --git a/shared/utils/server/stats.ts b/shared/utils/server/stats.ts
new file mode 100644
index 000000000..6f079ad18
--- /dev/null
+++ b/shared/utils/server/stats.ts
@@ -0,0 +1,22 @@
+import { makeGetRequest } from '../requests/requests'
+
+function getStats (url: string, useCache = false) {
+  const path = '/api/v1/server/stats'
+
+  const query = {
+    t: useCache ? undefined : new Date().getTime()
+  }
+
+  return makeGetRequest({
+    url,
+    path,
+    query,
+    statusCodeExpected: 200
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  getStats
+}