Merge branch 'develop' into pr/1217

8 files changed, 244 insertions, 72 deletions

diff --git a/server/tools/README.md b/server/tools/README.md
new file mode 100644
index 000000000..6b94d74e5
--- /dev/null
+++ b/server/tools/README.md
@@ -0,0 +1,82 @@
+peertube(8) -- companion CLI for PeerTube
+=========================================
+
+SYNOPSIS
+--------
+
+```
+peertube [command] [options]
+```
+
+DESCRIPTION
+-----------
+
+`peertube` wraps various utilities around PeerTube that are used either on a running local, running remote, or cold local instance.
+
+COMMANDS
+--------
+
+Unless otherwise specified, every command can be queried for its own help or manual by passing its name to the `help` command, or by using the `--help` option.
+
+`auth [action]`: stores credentials for your accounts on remote instances so that you don't need to pass them at every command
+
+`upload|up`: upload a video to a remote instance
+
+    $ peertube upload \
+        -u "PEERTUBE_URL" \
+        -U "PEERTUBE_USER" \
+        --password "PEERTUBE_PASSWORD"
+
+`import-videos|import`: import a video from a streaming platform to a remote instance
+
+    $ peertube import \
+        -u "PEERTUBE_URL" \
+        -U "PEERTUBE_USER" \
+        --password "PEERTUBE_PASSWORD" \
+        -t "TARGET_URL"
+
+    The target URL can be directly the video file, or any of the supported sites of youtube-dl. The video is downloaded locally and then uploaded. Already downloaded videos will not be uploaded twice, so you can run and re-run the script in case of crash, disconnection…
+
+`watch|w`: watch a video in the terminal ✩°｡⋆
+
+    -g, --gui <player>      player type (default: ascii)
+    -i, --invert            invert colors (ascii player only)
+    -r, --resolution <res>  video resolution (default: 720)
+
+    It provides support for different players:
+
+    - ascii (default ; plays in ascii art in your terminal!)
+    - mpv
+    - mplayer
+    - vlc
+    - stdout
+    - xbmc
+    - airplay
+    - chromecast
+
+`repl`: interact with the application libraries and objects even when PeerTube is not running
+
+    Type .help to see the repl-only functions, or to see the available PeerTube core functions:
+   
+    repl> lodash.keys(context)
+
+`help [cmd]`: display help for [cmd]
+
+EXAMPLES
+--------
+
+    $ peertube auth add -u "PEERTUBE_URL" -U "PEERTUBE_USER" --password "PEERTUBE_PASSWORD"
+    $ peertube up <videoFile>
+    $ peertube watch https://peertube.cpy.re/videos/watch/e8a1af4e-414a-4d58-bfe6-2146eed06d10
+
+SEE ALSO
+--------
+
+[PeerTube Tools Documentation](https://github.com/Chocobozzz/PeerTube/blob/develop/support/doc/tools.md)
+
+[PeerTube Admin Documentation](https://docs.joinpeertube.org/lang/en/docs/)
+
+REPORTING BUGS
+--------------
+
+See [PeerTube repository](https://github.com/Chocobozzz/PeerTube).
diff --git a/server/tools/cli.ts b/server/tools/cli.ts
index 53b20964e..108c44218 100644
--- a/server/tools/cli.ts
+++ b/server/tools/cli.ts
@@ -23,7 +23,7 @@ async function getSettings () {
       if (err) {
         return rej(err)
       }
-      return res(data || settings)
+      return res(Object.keys(data).length === 0 ? settings : data)
     })
   })
 }
diff --git a/server/tools/peertube-auth.ts b/server/tools/peertube-auth.ts
index 33438811e..a962944a4 100644
--- a/server/tools/peertube-auth.ts
+++ b/server/tools/peertube-auth.ts
@@ -5,34 +5,25 @@ import { getSettings, writeSettings, netrc } from './cli'
 import { isHostValid } from '../helpers/custom-validators/servers'
 import { isUserUsernameValid } from '../helpers/custom-validators/users'
 
-function delInstance (url: string) {
-  return new Promise((res, rej): void => {
-    getSettings()
-      .then(async (settings) => {
-        settings.remotes.splice(settings.remotes.indexOf(url))
-        await writeSettings(settings)
-        delete netrc.machines[url]
-        netrc.save()
-        res()
-      })
-      .catch(err => rej(err))
-  })
+async function delInstance (url: string) {
+  const settings = await getSettings()
+
+  settings.remotes.splice(settings.remotes.indexOf(url))
+  await writeSettings(settings)
+
+  delete netrc.machines[url]
+  await netrc.save()
 }
 
 async function setInstance (url: string, username: string, password: string) {
-  return new Promise((res, rej): void => {
-    getSettings()
-      .then(async settings => {
-        if (settings.remotes.indexOf(url) === -1) {
-          settings.remotes.push(url)
-        }
-        await writeSettings(settings)
-        netrc.machines[url] = { login: username, password }
-        netrc.save()
-        res()
-      })
-      .catch(err => rej(err))
-  })
+  const settings = await getSettings()
+  if (settings.remotes.indexOf(url) === -1) {
+    settings.remotes.push(url)
+  }
+  await writeSettings(settings)
+
+  netrc.machines[url] = { login: username, password }
+  await netrc.save()
 }
 
 function isURLaPeerTubeInstance (url: string) {
@@ -71,56 +62,60 @@ program
           required: true
         }
       }
-    }, (_, result) => {
-      setInstance(result.url, result.username, result.password)
+    }, async (_, result) => {
+      await setInstance(result.url, result.username, result.password)
+
+      process.exit(0)
     })
   })
 
 program
   .command('del <url>')
   .description('unregisters a remote instance')
-  .action((url) => {
-    delInstance(url)
+  .action(async url => {
+    await delInstance(url)
+
+    process.exit(0)
   })
 
 program
   .command('list')
   .description('lists registered remote instances')
-  .action(() => {
-    getSettings()
-      .then(settings => {
-        const table = new Table({
-          head: ['instance', 'login'],
-          colWidths: [30, 30]
-        })
-        netrc.loadSync()
-        settings.remotes.forEach(element => {
-          table.push([
-            element,
-            netrc.machines[element].login
-          ])
-        })
-
-        console.log(table.toString())
-      })
+  .action(async () => {
+    const settings = await getSettings()
+    const table = new Table({
+      head: ['instance', 'login'],
+      colWidths: [30, 30]
+    })
+    netrc.loadSync()
+    settings.remotes.forEach(element => {
+      table.push([
+        element,
+        netrc.machines[element].login
+      ])
+    })
+
+    console.log(table.toString())
+
+    process.exit(0)
   })
 
 program
   .command('set-default <url>')
   .description('set an existing entry as default')
-  .action((url) => {
-    getSettings()
-      .then(settings => {
-        const instanceExists = settings.remotes.indexOf(url) !== -1
-
-        if (instanceExists) {
-          settings.default = settings.remotes.indexOf(url)
-          writeSettings(settings)
-        } else {
-          console.log('<url> is not a registered instance.')
-          process.exit(-1)
-        }
-      })
+  .action(async url => {
+    const settings = await getSettings()
+    const instanceExists = settings.remotes.indexOf(url) !== -1
+
+    if (instanceExists) {
+      settings.default = settings.remotes.indexOf(url)
+      await writeSettings(settings)
+
+      process.exit(0)
+    } else {
+      console.log('<url> is not a registered instance.')
+      process.exit(-1)
+    }
   })
 
 program.on('--help', function () {
diff --git a/server/tools/peertube-get-access-token.ts b/server/tools/peertube-get-access-token.ts
index eb2571a03..a68665f5b 100644
--- a/server/tools/peertube-get-access-token.ts
+++ b/server/tools/peertube-get-access-token.ts
@@ -6,7 +6,7 @@ import {
   Server,
   Client,
   User
-} from '../tests/utils/index'
+} from '../../shared/utils'
 
 program
   .option('-u, --url <url>', 'Server url')
diff --git a/server/tools/peertube-import-videos.ts b/server/tools/peertube-import-videos.ts
index 15f517cab..151c5a989 100644
--- a/server/tools/peertube-import-videos.ts
+++ b/server/tools/peertube-import-videos.ts
@@ -6,10 +6,11 @@ import { join } from 'path'
 import { VideoPrivacy } from '../../shared/models/videos'
 import { doRequestAndSaveToFile } from '../helpers/requests'
 import { CONSTRAINTS_FIELDS } from '../initializers'
-import { getClient, getVideoCategories, login, searchVideoWithSort, uploadVideo } from '../tests/utils'
+import { getClient, getVideoCategories, login, searchVideoWithSort, uploadVideo } from '../../shared/utils/index'
 import { truncate } from 'lodash'
 import * as prompt from 'prompt'
 import { remove } from 'fs-extra'
+import { sha256 } from '../helpers/core-utils'
 import { safeGetYoutubeDL } from '../helpers/youtube-dl'
 import { getSettings, netrc } from './cli'
 
@@ -57,6 +58,7 @@ getSettings()
         settings.remotes[settings.default] :
         settings.remotes[0]
     }
+
     if (!program['username']) program['username'] = netrc.machines[program['url']].login
     if (!program['password']) program['password'] = netrc.machines[program['url']].password
   }
@@ -68,12 +70,19 @@ getSettings()
     process.exit(-1)
   }
 
+  removeEndSlashes(program['url'])
+  removeEndSlashes(program['targetUrl'])
+
   const user = {
     username: program['username'],
     password: program['password']
   }
 
-  run(user, program['url']).catch(err => console.error(err))
+  run(user, program['url'])
+    .catch(err => {
+      console.error(err)
+      process.exit(-1)
+    })
 })
 
 async function promptPassword () {
@@ -107,8 +116,12 @@ async function run (user, url: string) {
     secret: res.body.client_secret
   }
 
-  const res2 = await login(url, client, user)
-  accessToken = res2.body.access_token
+  try {
+    const res = await login(program[ 'url' ], client, user)
+    accessToken = res.body.access_token
+  } catch (err) {
+    throw new Error('Cannot authenticate. Please check your username/password.')
+  }
 
   const youtubeDL = await safeGetYoutubeDL()
 
@@ -133,8 +146,7 @@ async function run (user, url: string) {
       await processVideo(info, program['language'], processOptions.cwd, url, user)
     }
 
-    // https://www.youtube.com/watch?v=2Upx39TBc1s
-    console.log('I\'m finished!')
+    console.log('Video/s for user %s imported: %s', program['username'], program['targetUrl'])
     process.exit(0)
   })
 }
@@ -155,7 +167,7 @@ function processVideo (info: any, languageCode: string, cwd: string, url: string
       return res()
     }
 
-    const path = join(cwd, new Date().getTime() + '.mp4')
+    const path = join(cwd, sha256(videoInfo.url) + '.mp4')
 
     console.log('Downloading video "%s"...', videoInfo.title)
 
@@ -192,7 +204,7 @@ async function uploadVideoOnPeerTube (videoInfo: any, videoPath: string, cwd: st
 
   let thumbnailfile
   if (videoInfo.thumbnail) {
-    thumbnailfile = join(cwd, 'thumbnail.jpg')
+    thumbnailfile = join(cwd, sha256(videoInfo.thumbnail) + '.jpg')
 
     await doRequestAndSaveToFile({
       method: 'GET',
@@ -322,3 +334,9 @@ function isNSFW (info: any) {
 
   return false
 }
+
+function removeEndSlashes (url: string) {
+  while (url.endsWith('/')) {
+    url.slice(0, -1)
+  }
+}
diff --git a/server/tools/peertube-repl.ts b/server/tools/peertube-repl.ts
new file mode 100644
index 000000000..04d8b95a3
--- /dev/null
+++ b/server/tools/peertube-repl.ts
@@ -0,0 +1,76 @@
+import * as repl from 'repl'
+import * as path from 'path'
+import * as _ from 'lodash'
+import * as uuidv1 from 'uuid/v1'
+import * as uuidv3 from 'uuid/v3'
+import * as uuidv4 from 'uuid/v4'
+import * as uuidv5 from 'uuid/v5'
+import * as Sequelize from 'sequelize'
+import * as YoutubeDL from 'youtube-dl'
+
+import { initDatabaseModels, sequelizeTypescript } from '../initializers'
+import * as cli from '../tools/cli'
+import { logger } from '../helpers/logger'
+import * as constants from '../initializers/constants'
+import * as modelsUtils from '../models/utils'
+import * as coreUtils from '../helpers/core-utils'
+import * as ffmpegUtils from '../helpers/ffmpeg-utils'
+import * as peertubeCryptoUtils from '../helpers/peertube-crypto'
+import * as signupUtils from '../helpers/signup'
+import * as utils from '../helpers/utils'
+import * as YoutubeDLUtils from '../helpers/youtube-dl'
+
+const start = async () => {
+  await initDatabaseModels(true)
+
+  const versionCommitHash = await utils.getServerCommit()
+
+  const initContext = (replServer) => {
+    return (context) => {
+      const properties = {
+        context, repl: replServer, env: process.env,
+        lodash: _, path,
+        uuidv1, uuidv3, uuidv4, uuidv5,
+        cli, logger, constants,
+        Sequelize, sequelizeTypescript, modelsUtils,
+        models: sequelizeTypescript.models, transaction: sequelizeTypescript.transaction,
+        query: sequelizeTypescript.query, queryInterface: sequelizeTypescript.getQueryInterface(),
+        YoutubeDL,
+        coreUtils, ffmpegUtils, peertubeCryptoUtils, signupUtils, utils, YoutubeDLUtils
+      }
+
+      for (let prop in properties) {
+        Object.defineProperty(context, prop, {
+          configurable: false,
+          enumerable: true,
+          value: properties[prop]
+        })
+      }
+    }
+  }
+
+  const replServer = repl.start({
+    prompt: `PeerTube [${cli.version}] (${versionCommitHash})> `
+  })
+
+  initContext(replServer)(replServer.context)
+  replServer.on('reset', initContext(replServer))
+  replServer.on('exit', () => process.exit())
+
+  const resetCommand = {
+    help: 'Reset REPL',
+    action () {
+      this.write('.clear\n')
+      this.displayPrompt()
+    }
+  }
+  replServer.defineCommand('reset', resetCommand)
+  replServer.defineCommand('r', resetCommand)
+
+}
+
+start().then((data) => {
+  // do nothing
+}).catch((err) => {
+  console.error(err)
+})
diff --git a/server/tools/peertube-upload.ts b/server/tools/peertube-upload.ts
index b17bc4288..ebc62c965 100644
--- a/server/tools/peertube-upload.ts
+++ b/server/tools/peertube-upload.ts
@@ -1,8 +1,8 @@
 import * as program from 'commander'
 import { access, constants } from 'fs-extra'
 import { isAbsolute } from 'path'
-import { getClient, login } from '../tests/utils'
-import { uploadVideo } from '../tests/utils/index'
+import { getClient, login } from '../../shared/utils'
+import { uploadVideo } from '../../shared/utils/'
 import { VideoPrivacy } from '../../shared/models/videos'
 import { netrc, getSettings } from './cli'
 
diff --git a/server/tools/peertube.ts b/server/tools/peertube.ts
index ad76bafb4..5d3ab2815 100755
--- a/server/tools/peertube.ts
+++ b/server/tools/peertube.ts
@@ -17,6 +17,7 @@ program
   .command('import-videos', 'import a video from a streaming platform').alias('import')
   .command('get-access-token', 'get a peertube access token', { noHelp: true }).alias('token')
   .command('watch', 'watch a video in the terminal ✩°｡⋆').alias('w')
+  .command('repl', 'initiate a REPL to access internals')
 
 /* Not Yet Implemented */
 program
@@ -57,7 +58,7 @@ if (!process.argv.slice(2).length) {
             ,"\\/
       _,.__/"\\/_                     (the CLI for red chocobos)
      / \\) "./,  ".
-  --/---"---" "-) )---- by Chocobozzz et al.`)
+  --/---"---" "-) )---- by Chocobozzz et al.\n`)
 }
 
 getSettings()