Add redundancy CLI

8 files changed, 308 insertions, 41 deletions

diff --git a/server/tests/cli/peertube.ts b/server/tests/cli/peertube.ts
index b8c0b1f79..09cbcdb65 100644
--- a/server/tests/cli/peertube.ts
+++ b/server/tests/cli/peertube.ts
@@ -6,15 +6,15 @@ import {
   addVideoChannel,
   buildAbsoluteFixturePath,
   cleanupTests,
-  createUser,
+  createUser, doubleFollow,
   execCLI,
   flushAndRunServer,
-  getEnvCli,
+  getEnvCli, getLocalIdByUUID,
   getVideo,
   getVideosList,
   getVideosListWithToken, removeVideo,
   ServerInfo,
-  setAccessTokensToServers,
+  setAccessTokensToServers, uploadVideo, uploadVideoAndGetId,
   userLogin,
   waitJobs
 } from '../../../shared/extra-utils'
@@ -210,6 +210,75 @@ describe('Test CLI wrapper', function () {
     })
   })
 
+  describe('Manage video redundancies', function () {
+    let anotherServer: ServerInfo
+    let video1Server2: number
+    let servers: ServerInfo[]
+
+    before(async function () {
+      this.timeout(120000)
+
+      anotherServer = await flushAndRunServer(2)
+      await setAccessTokensToServers([ anotherServer ])
+
+      await doubleFollow(server, anotherServer)
+
+      servers = [ server, anotherServer ]
+      await waitJobs(servers)
+
+      const uuid = (await uploadVideoAndGetId({ server: anotherServer, videoName: 'super video' })).uuid
+      await waitJobs(servers)
+
+      video1Server2 = await getLocalIdByUUID(server.url, uuid)
+    })
+
+    it('Should add a redundancy', async function () {
+      this.timeout(60000)
+
+      const env = getEnvCli(server)
+
+      const params = `add --video ${video1Server2}`
+
+      await execCLI(`${env} ${cmd} redundancy ${params}`)
+
+      await waitJobs(servers)
+    })
+
+    it('Should list redundancies', async function () {
+      this.timeout(60000)
+
+      {
+        const env = getEnvCli(server)
+
+        const params = `list-my-redundancies`
+        const stdout = await execCLI(`${env} ${cmd} redundancy ${params}`)
+
+        expect(stdout).to.contain('super video')
+        expect(stdout).to.contain(`localhost:${server.port}`)
+      }
+    })
+
+    it('Should remove a redundancy', async function () {
+      this.timeout(60000)
+
+      const env = getEnvCli(server)
+
+      const params = `remove --video ${video1Server2}`
+
+      await execCLI(`${env} ${cmd} redundancy ${params}`)
+
+      await waitJobs(servers)
+
+      {
+        const env = getEnvCli(server)
+        const params = `list-my-redundancies`
+        const stdout = await execCLI(`${env} ${cmd} redundancy ${params}`)
+
+        expect(stdout).to.not.contain('super video')
+      }
+    })
+  })
+
   after(async function () {
     this.timeout(10000)
 
diff --git a/server/tools/cli.ts b/server/tools/cli.ts
index 58e2445ac..ba80872fb 100644
--- a/server/tools/cli.ts
+++ b/server/tools/cli.ts
@@ -6,6 +6,8 @@ import { getVideoChannel } from '../../shared/extra-utils/videos/video-channels'
 import { Command } from 'commander'
 import { VideoChannel, VideoPrivacy } from '../../shared/models/videos'
 import { createLogger, format, transports } from 'winston'
+import { getAccessToken, getMyUserInformation } from '@shared/extra-utils'
+import { User, UserRole } from '@shared/models'
 
 let configName = 'PeerTube/CLI'
 if (isTestInstance()) configName += `-${getAppNumber()}`
@@ -14,6 +16,19 @@ const config = require('application-config')(configName)
 
 const version = require('../../../package.json').version
 
+async function getAdminTokenOrDie (url: string, username: string, password: string) {
+  const accessToken = await getAccessToken(url, username, password)
+  const resMe = await getMyUserInformation(url, accessToken)
+  const me: User = resMe.body
+
+  if (me.role !== UserRole.ADMINISTRATOR) {
+    console.error('You must be an administrator.')
+    process.exit(-1)
+  }
+
+  return accessToken
+}
+
 interface Settings {
   remotes: any[],
   default: number
@@ -222,5 +237,7 @@ export {
   getServerCredentials,
 
   buildCommonVideoOptions,
-  buildVideoAttributesFromCommander
+  buildVideoAttributesFromCommander,
+
+  getAdminTokenOrDie
 }
diff --git a/server/tools/package.json b/server/tools/package.json
index 40959d76e..06ad31cab 100644
--- a/server/tools/package.json
+++ b/server/tools/package.json
@@ -4,11 +4,12 @@
   "private": true,
   "dependencies": {
     "application-config": "^1.0.1",
-    "cli-table": "^0.3.1",
+    "cli-table3": "^0.5.1",
     "netrc-parser": "^3.1.6",
     "webtorrent-hybrid": "^4.0.1"
   },
   "summon": {
     "silent": true
-  }
+  },
+  "devDependencies": {}
 }
diff --git a/server/tools/peertube-auth.ts b/server/tools/peertube-auth.ts
index 6597a5c36..acac75034 100644
--- a/server/tools/peertube-auth.ts
+++ b/server/tools/peertube-auth.ts
@@ -6,8 +6,7 @@ import * as prompt from 'prompt'
 import { getNetrc, getSettings, writeSettings } from './cli'
 import { isUserUsernameValid } from '../helpers/custom-validators/users'
 import { getAccessToken, login } from '../../shared/extra-utils'
-
-const Table = require('cli-table')
+import * as CliTable3 from 'cli-table3'
 
 async function delInstance (url: string) {
   const [ settings, netrc ] = await Promise.all([ getSettings(), getNetrc() ])
@@ -108,10 +107,10 @@ program
   .action(async () => {
     const [ settings, netrc ] = await Promise.all([ getSettings(), getNetrc() ])
 
-    const table = new Table({
+    const table = new CliTable3({
       head: ['instance', 'login'],
       colWidths: [30, 30]
-    })
+    }) as CliTable3.HorizontalTable
 
     settings.remotes.forEach(element => {
       if (!netrc.machines[element]) return
diff --git a/server/tools/peertube-plugins.ts b/server/tools/peertube-plugins.ts
index e40606107..b341c14c1 100644
--- a/server/tools/peertube-plugins.ts
+++ b/server/tools/peertube-plugins.ts
@@ -3,15 +3,11 @@ registerTSPaths()
 
 import * as program from 'commander'
 import { PluginType } from '../../shared/models/plugins/plugin.type'
-import { getAccessToken } from '../../shared/extra-utils/users/login'
-import { getMyUserInformation } from '../../shared/extra-utils/users/users'
 import { installPlugin, listPlugins, uninstallPlugin, updatePlugin } from '../../shared/extra-utils/server/plugins'
-import { getServerCredentials } from './cli'
-import { User, UserRole } from '../../shared/models/users'
+import { getAdminTokenOrDie, getServerCredentials } from './cli'
 import { PeerTubePlugin } from '../../shared/models/plugins/peertube-plugin.model'
 import { isAbsolute } from 'path'
-
-const Table = require('cli-table')
+import * as CliTable3 from 'cli-table3'
 
 program
   .name('plugins')
@@ -82,10 +78,10 @@ async function pluginsListCLI () {
   })
   const plugins: PeerTubePlugin[] = res.body.data
 
-  const table = new Table({
+  const table = new CliTable3({
     head: ['name', 'version', 'homepage'],
     colWidths: [ 50, 10, 50 ]
-  })
+  }) as CliTable3.HorizontalTable
 
   for (const plugin of plugins) {
     const npmName = plugin.type === PluginType.PLUGIN
@@ -192,16 +188,3 @@ async function uninstallPluginCLI (options: any) {
   console.log('Plugin uninstalled.')
   process.exit(0)
 }
-
-async function getAdminTokenOrDie (url: string, username: string, password: string) {
-  const accessToken = await getAccessToken(url, username, password)
-  const resMe = await getMyUserInformation(url, accessToken)
-  const me: User = resMe.body
-
-  if (me.role !== UserRole.ADMINISTRATOR) {
-    console.error('Cannot list plugins if you are not administrator.')
-    process.exit(-1)
-  }
-
-  return accessToken
-}
diff --git a/server/tools/peertube-redundancy.ts b/server/tools/peertube-redundancy.ts
new file mode 100644
index 000000000..a71f48104
--- /dev/null
+++ b/server/tools/peertube-redundancy.ts
@@ -0,0 +1,194 @@
+import { registerTSPaths } from '../helpers/register-ts-paths'
+registerTSPaths()
+
+import * as program from 'commander'
+import { getAdminTokenOrDie, getServerCredentials } from './cli'
+import { VideoRedundanciesTarget, VideoRedundancy } from '@shared/models'
+import { addVideoRedundancy, listVideoRedundancies, removeVideoRedundancy } from '@shared/extra-utils/server/redundancy'
+import validator from 'validator'
+import bytes = require('bytes')
+import * as CliTable3 from 'cli-table3'
+import { parse } from 'url'
+import { uniq } from 'lodash'
+
+program
+  .name('plugins')
+  .usage('[command] [options]')
+
+program
+  .command('list-remote-redundancies')
+  .description('List remote redundancies on your videos')
+  .option('-u, --url <url>', 'Server url')
+  .option('-U, --username <username>', 'Username')
+  .option('-p, --password <token>', 'Password')
+  .action(() => listRedundanciesCLI('my-videos'))
+
+program
+  .command('list-my-redundancies')
+  .description('List your redundancies of remote videos')
+  .option('-u, --url <url>', 'Server url')
+  .option('-U, --username <username>', 'Username')
+  .option('-p, --password <token>', 'Password')
+  .action(() => listRedundanciesCLI('remote-videos'))
+
+program
+  .command('add')
+  .description('Duplicate a video in your redundancy system')
+  .option('-u, --url <url>', 'Server url')
+  .option('-U, --username <username>', 'Username')
+  .option('-p, --password <token>', 'Password')
+  .option('-v, --video <videoId>', 'Video id to duplicate')
+  .action((options) => addRedundancyCLI(options))
+
+program
+  .command('remove')
+  .description('Remove a video from your redundancies')
+  .option('-u, --url <url>', 'Server url')
+  .option('-U, --username <username>', 'Username')
+  .option('-p, --password <token>', 'Password')
+  .option('-v, --video <videoId>', 'Video id to remove from redundancies')
+  .action((options) => removeRedundancyCLI(options))
+
+if (!process.argv.slice(2).length) {
+  program.outputHelp()
+}
+
+program.parse(process.argv)
+
+// ----------------------------------------------------------------------------
+
+async function listRedundanciesCLI (target: VideoRedundanciesTarget) {
+  const { url, username, password } = await getServerCredentials(program)
+  const accessToken = await getAdminTokenOrDie(url, username, password)
+
+  const redundancies = await listVideoRedundanciesData(url, accessToken, target)
+
+  const table = new CliTable3({
+    head: [ 'video id', 'video name', 'video url', 'files', 'playlists', 'by instances', 'total size' ]
+  }) as CliTable3.HorizontalTable
+
+  for (const redundancy of redundancies) {
+    const webtorrentFiles = redundancy.redundancies.files
+    const streamingPlaylists = redundancy.redundancies.streamingPlaylists
+
+    let totalSize = ''
+    if (target === 'remote-videos') {
+      const tmp = webtorrentFiles.concat(streamingPlaylists)
+                                 .reduce((a, b) => a + b.size, 0)
+
+      totalSize = bytes(tmp)
+    }
+
+    const instances = uniq(
+      webtorrentFiles.concat(streamingPlaylists)
+                     .map(r => r.fileUrl)
+                     .map(u => parse(u).host)
+    )
+
+    table.push([
+      redundancy.id.toString(),
+      redundancy.name,
+      redundancy.url,
+      webtorrentFiles.length,
+      streamingPlaylists.length,
+      instances.join('\n'),
+      totalSize
+    ])
+  }
+
+  console.log(table.toString())
+  process.exit(0)
+}
+
+async function addRedundancyCLI (options: { videoId: number }) {
+  const { url, username, password } = await getServerCredentials(program)
+  const accessToken = await getAdminTokenOrDie(url, username, password)
+
+  if (!options[ 'video' ] || validator.isInt('' + options[ 'video' ]) === false) {
+    console.error('You need to specify the video id to duplicate and it should be a number.\n')
+    program.outputHelp()
+    process.exit(-1)
+  }
+
+  try {
+    await addVideoRedundancy({
+      url,
+      accessToken,
+      videoId: options[ 'video' ]
+    })
+
+    console.log('Video will be duplicated by your instance!')
+
+    process.exit(0)
+  } catch (err) {
+    if (err.message.includes(409)) {
+      console.error('This video is already duplicated by your instance.')
+    } else if (err.message.includes(404)) {
+      console.error('This video id does not exist.')
+    } else {
+      console.error(err)
+    }
+
+    process.exit(-1)
+  }
+}
+
+async function removeRedundancyCLI (options: { videoId: number }) {
+  const { url, username, password } = await getServerCredentials(program)
+  const accessToken = await getAdminTokenOrDie(url, username, password)
+
+  if (!options[ 'video' ] || validator.isInt('' + options[ 'video' ]) === false) {
+    console.error('You need to specify the video id to remove from your redundancies.\n')
+    program.outputHelp()
+    process.exit(-1)
+  }
+
+  const videoId = parseInt(options[ 'video' ] + '', 10)
+
+  let redundancies = await listVideoRedundanciesData(url, accessToken, 'my-videos')
+  let videoRedundancy = redundancies.find(r => videoId === r.id)
+
+  if (!videoRedundancy) {
+    redundancies = await listVideoRedundanciesData(url, accessToken, 'remote-videos')
+    videoRedundancy = redundancies.find(r => videoId === r.id)
+  }
+
+  if (!videoRedundancy) {
+    console.error('Video redundancy not found.')
+    process.exit(-1)
+  }
+
+  try {
+    const ids = videoRedundancy.redundancies.files
+                               .concat(videoRedundancy.redundancies.streamingPlaylists)
+                               .map(r => r.id)
+
+    for (const id of ids) {
+      await removeVideoRedundancy({
+        url,
+        accessToken,
+        redundancyId: id
+      })
+    }
+
+    console.log('Video redundancy removed!')
+
+    process.exit(0)
+  } catch (err) {
+    console.error(err)
+    process.exit(-1)
+  }
+}
+
+async function listVideoRedundanciesData (url: string, accessToken: string, target: VideoRedundanciesTarget) {
+  const res = await listVideoRedundancies({
+    url,
+    accessToken,
+    start: 0,
+    count: 100,
+    sort: 'name',
+    target
+  })
+
+  return res.body.data as VideoRedundancy[]
+}
diff --git a/server/tools/peertube.ts b/server/tools/peertube.ts
index fc85c4210..9883bbf05 100644
--- a/server/tools/peertube.ts
+++ b/server/tools/peertube.ts
@@ -22,6 +22,7 @@ program
   .command('watch', 'watch a video in the terminal ✩°｡⋆').alias('w')
   .command('repl', 'initiate a REPL to access internals')
   .command('plugins [action]', 'manage instance plugins/themes').alias('p')
+  .command('redundancy [action]', 'manage instance redundancies').alias('r')
 
 /* Not Yet Implemented */
 program
diff --git a/server/tools/yarn.lock b/server/tools/yarn.lock
index 28756cbc2..ccd716a51 100644
--- a/server/tools/yarn.lock
+++ b/server/tools/yarn.lock
@@ -347,12 +347,15 @@ chunk-store-stream@^4.0.0:
     block-stream2 "^2.0.0"
     readable-stream "^3.4.0"
 
-cli-table@^0.3.1:
-  version "0.3.1"
-  resolved "https://registry.yarnpkg.com/cli-table/-/cli-table-0.3.1.tgz#f53b05266a8b1a0b934b3d0821e6e2dc5914ae23"
-  integrity sha1-9TsFJmqLGguTSz0IIebi3FkUriM=
+cli-table3@^0.5.1:
+  version "0.5.1"
+  resolved "https://registry.yarnpkg.com/cli-table3/-/cli-table3-0.5.1.tgz#0252372d94dfc40dbd8df06005f48f31f656f202"
+  integrity sha512-7Qg2Jrep1S/+Q3EceiZtQcDPWxhAvBw+ERf1162v4sikJrvojMHFqXt8QIVha8UlH9rgU0BeWPytZ9/TzYqlUw==
   dependencies:
-    colors "1.0.3"
+    object-assign "^4.1.0"
+    string-width "^2.1.1"
+  optionalDependencies:
+    colors "^1.1.2"
 
 clivas@^0.2.0:
   version "0.2.0"
@@ -364,10 +367,10 @@ code-point-at@^1.0.0:
   resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77"
   integrity sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=
 
-colors@1.0.3:
-  version "1.0.3"
-  resolved "https://registry.yarnpkg.com/colors/-/colors-1.0.3.tgz#0433f44d809680fdeb60ed260f1b0c262e82a40b"
-  integrity sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=
+colors@^1.1.2:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78"
+  integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==
 
 common-tags@^1.8.0:
   version "1.8.0"
@@ -1609,7 +1612,7 @@ string-width@^1.0.1:
     is-fullwidth-code-point "^1.0.0"
     strip-ansi "^3.0.0"
 
-"string-width@^1.0.2 || 2":
+"string-width@^1.0.2 || 2", string-width@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-2.1.1.tgz#ab93f27a8dc13d28cac815c462143a6d9012ae9e"
   integrity sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==