9 files changed, 285 insertions, 20 deletions

diff --git a/support/doc/tools.md b/support/doc/tools.md
index 1db29edc0..7f93c94f2 100644
--- a/support/doc/tools.md
+++ b/support/doc/tools.md
@@ -1,3 +1,30 @@
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**  *generated with [DocToc](https://github.com/thlorenz/doctoc)*
+
+- [CLI tools guide](#cli-tools-guide)
+  - [CLI wrapper](#cli-wrapper)
+  - [Remote Tools](#remote-tools)
+    - [Dependencies](#dependencies)
+    - [Installation](#installation)
+    - [peertube-import-videos.js](#peertube-import-videosjs)
+    - [peertube-upload.js](#peertube-uploadjs)
+    - [peertube-watch.js](#peertube-watchjs)
+  - [Server tools](#server-tools)
+    - [parse-log](#parse-log)
+    - [create-transcoding-job.js](#create-transcoding-jobjs)
+    - [create-import-video-file-job.js](#create-import-video-file-jobjs)
+    - [prune-storage.js](#prune-storagejs)
+    - [optimize-old-videos.js](#optimize-old-videosjs)
+    - [update-host.js](#update-hostjs)
+    - [REPL (Read Eval Print Loop)](#repl-read-eval-print-loop)
+      - [.help](#help)
+      - [Lodash example](#lodash-example)
+      - [YoutubeDL example](#youtubedl-example)
+      - [Models examples](#models-examples)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
 # CLI tools guide
  - [CLI wrapper](#cli-wrapper)
  - [Remote tools](#remote-tools)
@@ -159,7 +186,7 @@ $ sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production
 
 ### create-transcoding-job.js
 
-You can use this script to force transcoding of an existing video.
+You can use this script to force transcoding of an existing video. PeerTube needs to be running.
 
 ```
 $ sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run create-transcoding-job -- -v [videoUUID]
@@ -172,7 +199,7 @@ $ sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production
    
 ### create-import-video-file-job.js
 
-You can use this script to import a video file to replace an already uploaded file or to add a new resolution to a video.
+You can use this script to import a video file to replace an already uploaded file or to add a new resolution to a video. PeerTube needs to be running.
 
 ```
 $ sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run create-import-video-file-job -- -v [videoUUID] -i [videoFile]
@@ -187,11 +214,155 @@ To delete them (a confirmation will be demanded first):
 $ sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run prune-storage
 ```
 
+### optimize-old-videos.js
+
+Before version v1.0.0-beta.16, Peertube did not specify a bitrate for the
+transcoding of uploaded videos. This means that videos might be encoded into
+very large files that are too large for streaming. This script re-transcodes
+these videos so that they can be watched properly, even on slow connections.
+
+```
+$ sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run optimize-old-videos
+```
+
+
 ### update-host.js
 
-If you started PeerTube with a domain, and then changed it you will have invalid torrent files and invalid URLs in your database.
-To fix this, you have to run:
+If you started PeerTube with a domain, and then changed it you will have
+invalid torrent files and invalid URLs in your database. To fix this, you have
+to run:
 
 ```
 $ sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run update-host
 ```
+
+### REPL ([Read Eval Print Loop](https://nodejs.org/docs/latest-v8.x/api/repl.html))
+
+If you want to interact with the application libraries and objects even when PeerTube is not running, there is a REPL for that.
+
+usage: `node ./dist/server/tools/repl.js`
+
+"The default evaluator will, by default, assign the result of the most recently evaluated expression to the special variable `_` (underscore). Explicitly setting `_` to a value will disable this behavior."
+
+- type `.help` to list commands available in the repl, notice it starts with a dot
+- type `.exit` to exit, note that you still have to press CTRL-C to actually exit, or press CTRL-C (3 times) without typing `.exit` to exit
+- type `context` to list all available objects and libraries in the context, note: `Promise` is also available but it's not listed in the context, in case you need promises for something
+- type `env` to see the loaded environment variables
+- type `path` to access path library
+- type `lodash` to access lodash library
+- type `uuidv1` to access uuid/v1 library
+- type `uuidv3` to access uuid/v3 library
+- type `uuidv4` to access uuid/v4 library
+- type `uuidv5` to access uuid/v5 library
+- type `YoutubeDL` to access youtube-dl library
+- type `cli` to access the cli helpers object
+- type `logger` to access the logger; if you log to it, it will write to stdout and to the peertube.log file
+- type `constants` to access the constants loaded by the server
+- type `coreUtils` to access the core-utils helpers object
+- type `ffmpegUtils` to access the ffmpeg-utils helpers object
+- type `peertubeCryptoUtils` to access the peertube-crypto helpers object
+- type `signupUtils` to access the signup helpers object
+- type `utils` to access the utils helpers object
+- type `YoutubeDLUtils` to access the youtube-dl helpers object
+- type `sequelizeTypescript` to access sequelizeTypescript
+- type `modelsUtils` to access the models/utils
+- type `models` to access the shortcut to sequelizeTypescript.models
+- type `transaction` to access the shortcut to sequelizeTypescript.transaction
+- type `query` to access the shortcut to sequelizeTypescript.query
+- type `queryInterface` to access the shortcut to sequelizeTypescript.queryInterface
+
+#### .help
+
+```
+PeerTube [1.0.0] (b10eb595)> .help
+.break    Sometimes you get stuck, this gets you out
+.clear    Break, and also clear the local context
+.editor   Enter editor mode
+.exit     Exit the repl
+.help     Print this help message
+.load     Load JS from a file into the REPL session
+.r        Reset REPL
+.reset    Reset REPL
+.save     Save all evaluated commands in this REPL session to a file
+PeerTube [1.0.0] (b10eb595)>
+```
+
+#### Lodash example
+
+```
+PeerTube [1.0.0] (b10eb595)> lodash.keys(context)
+[ 'global',
+  'console',
+  'DTRACE_NET_SERVER_CONNECTION',
+  'DTRACE_NET_STREAM_END',
+  'DTRACE_HTTP_SERVER_REQUEST',
+  'DTRACE_HTTP_SERVER_RESPONSE',
+  'DTRACE_HTTP_CLIENT_REQUEST',
+  'DTRACE_HTTP_CLIENT_RESPONSE',
+  'process',
+  'Buffer',
+  'clearImmediate',
+  'clearInterval',
+  'clearTimeout',
+  'setImmediate',
+  'setInterval',
+  'setTimeout',
+  'XMLHttpRequest',
+  'compact2string',
+  'module',
+  'require',
+  'path',
+  'repl',
+  'context',
+  'env',
+  'lodash',
+  'uuidv1',
+  'uuidv3',
+  'uuidv4',
+  'uuidv5',
+  'cli',
+  'logger',
+  'constants',
+  'Sequelize',
+  'sequelizeTypescript',
+  'modelsUtils',
+  'models',
+  'transaction',
+  'query',
+  'queryInterface',
+  'YoutubeDL',
+  'coreUtils',
+  'ffmpegUtils',
+  'peertubeCryptoUtils',
+  'signupUtils',
+  'utils',
+  'YoutubeDLUtils' ]
+PeerTube [1.0.0] (b10eb595)>
+```
+
+#### YoutubeDL example
+```
+YoutubeDL.getInfo('https://www.youtube.com/watch?v=I5ZN289jjDo', function(err, data) {console.log(err, data)})
+```
+
+#### Models examples
+```
+PeerTube [1.0.0] (b10eb595)> new models.ActorModel({id: 3}).getVideoChannel().then(function(data){console.log(data.dataValues.name)})
+Promise {
+  _bitField: 0,
+  _fulfillmentHandler0: undefined,
+  _rejectionHandler0: undefined,
+  _promise0: undefined,
+  _receiver0: undefined }
+PeerTube [1.0.0] (b10eb595)> Main root channel
+PeerTube [1.0.0] (b10eb595)> let out; new models.UserModel({id: 1}).getAccount().then(function (data) {out = data.dataValues.id})
+Promise {
+  _bitField: 0,
+  _fulfillmentHandler0: undefined,
+  _rejectionHandler0: undefined,
+  _promise0: undefined,
+  _receiver0: undefined }
+PeerTube [1.0.0] (b10eb595)> out
+2
+PeerTube [1.0.0] (b10eb595)>
+```
diff --git a/support/docker/dev/Dockerfile b/support/docker/dev/Dockerfile
index 2b4f2b215..aa4a8a3d6 100644
--- a/support/docker/dev/Dockerfile
+++ b/support/docker/dev/Dockerfile
@@ -1,32 +1,48 @@
-FROM janitortechnology/ubuntu-dev
+FROM ubuntu:bionic
+
+# Avoid tzdata interactive dialog
+ENV DEBIAN_FRONTEND=noninteractive
 
 # Install PeerTube's dependencies.
 # Packages are from https://github.com/Chocobozzz/PeerTube#dependencies
-RUN sudo apt-get update -q && sudo apt-get install -qy \
+RUN apt-get update -q && apt-get install -qy \
+ curl \
+ nano \
  ffmpeg \
  postgresql \
- openssl
+ postgresql-contrib \
+ openssl \
+ g++ \
+ make \
+ redis-server \
+ git \
+ gnupg
+
+# Install NodeJS 8.x
+RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - && \
+ apt-get install -y nodejs
+
+# Install Yarn
+RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \
+ echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \
+ apt-get update && apt-get install yarn
 
 # Download PeerTube's source code.
 RUN git clone -b develop https://github.com/Chocobozzz/PeerTube /home/user/PeerTube
 WORKDIR /home/user/PeerTube
 
-# Configure the IDEs to use Janitor's source directory as workspace.
-ENV WORKSPACE /home/user/PeerTube/
-
 # Install dependencies.
 RUN yarn install --pure-lockfile
 
-# Configure Janitor for PeerTube.
-COPY --chown=user:user janitor.json /home/user/
+# Configure and run PeerTube.
+COPY setup_postgres.sql /tmp/
+RUN service postgresql start \
+ && su postgres -c "psql --file=/tmp/setup_postgres.sql"
 
-# Configure and build PeerTube.
-COPY create_user.sql /tmp/
-RUN sudo service postgresql start \
- && sudo -u postgres psql --file=/tmp/create_user.sql \
- && npm run build
-
-COPY --chown=user:user supervisord.conf /tmp/supervisord-extra.conf
-RUN cat /tmp/supervisord-extra.conf | sudo tee -a /etc/supervisord.conf
+# Expose PeerTube sources as a volume
+VOLUME /home/user/PeerTube
 
 EXPOSE 3000 9000
+
+# Start PostgreSQL and Redis
+CMD service postgresql start && redis-server
diff --git a/support/docker/dev/setup_postgres.sql b/support/docker/dev/setup_postgres.sql
new file mode 100644
index 000000000..0937f9d19
--- /dev/null
+++ b/support/docker/dev/setup_postgres.sql
@@ -0,0 +1,6 @@
+create database peertube_dev;
+create user peertube password 'peertube';
+grant all privileges on database peertube_dev to peertube;
+\c peertube_dev
+CREATE EXTENSION pg_trgm;
+CREATE EXTENSION unaccent;
diff --git a/support/docker/dev/usage.md b/support/docker/dev/usage.md
new file mode 100644
index 000000000..319d7db30
--- /dev/null
+++ b/support/docker/dev/usage.md
@@ -0,0 +1,20 @@
+### Usage
+1. Build the image:
+        ```
+        docker build -t my_peertube_dev .
+        ```
+1. Start the container:
+        ```
+        docker run -d -i -p 3000:3000 -p 9000:9000 --name peertube my_peertube_dev
+        ```
+        This will create a new Docker volume containing PeerTube sources.
+
+1. Start PeerTube inside the container:
+        ```
+        docker exec -it peertube npm run dev
+        ```
+1. In another window, find the path to the Docker volume
+        ```
+        docker inspect peertube | less +/Mounts
+        ```
+        You can now make changes to the files. They should be automatically recompiled.
diff --git a/support/docker/janitor/Dockerfile b/support/docker/janitor/Dockerfile
new file mode 100644
index 000000000..2b4f2b215
--- /dev/null
+++ b/support/docker/janitor/Dockerfile
@@ -0,0 +1,32 @@
+FROM janitortechnology/ubuntu-dev
+
+# Install PeerTube's dependencies.
+# Packages are from https://github.com/Chocobozzz/PeerTube#dependencies
+RUN sudo apt-get update -q && sudo apt-get install -qy \
+ ffmpeg \
+ postgresql \
+ openssl
+
+# Download PeerTube's source code.
+RUN git clone -b develop https://github.com/Chocobozzz/PeerTube /home/user/PeerTube
+WORKDIR /home/user/PeerTube
+
+# Configure the IDEs to use Janitor's source directory as workspace.
+ENV WORKSPACE /home/user/PeerTube/
+
+# Install dependencies.
+RUN yarn install --pure-lockfile
+
+# Configure Janitor for PeerTube.
+COPY --chown=user:user janitor.json /home/user/
+
+# Configure and build PeerTube.
+COPY create_user.sql /tmp/
+RUN sudo service postgresql start \
+ && sudo -u postgres psql --file=/tmp/create_user.sql \
+ && npm run build
+
+COPY --chown=user:user supervisord.conf /tmp/supervisord-extra.conf
+RUN cat /tmp/supervisord-extra.conf | sudo tee -a /etc/supervisord.conf
+
+EXPOSE 3000 9000
diff --git a/support/docker/dev/create_user.sql b/support/docker/janitor/create_user.sql
index c2fbcf27e..c2fbcf27e 100644
--- a/support/docker/dev/create_user.sql
+++ b/support/docker/janitor/create_user.sql
diff --git a/support/docker/dev/janitor.json b/support/docker/janitor/janitor.json
index 5acdf3060..5acdf3060 100644
--- a/support/docker/dev/janitor.json
+++ b/support/docker/janitor/janitor.json
diff --git a/support/docker/dev/supervisord.conf b/support/docker/janitor/supervisord.conf
index b2e1682df..b2e1682df 100644
--- a/support/docker/dev/supervisord.conf
+++ b/support/docker/janitor/supervisord.conf
diff --git a/support/docker/production/config/custom-environment-variables.yaml b/support/docker/production/config/custom-environment-variables.yaml
index daf885813..1f7fbf849 100644
--- a/support/docker/production/config/custom-environment-variables.yaml
+++ b/support/docker/production/config/custom-environment-variables.yaml
@@ -56,6 +56,26 @@ signup:
     __name: "PEERTUBE_SIGNUP_LIMIT"
     __format: "json"
 
+search:
+  remote_uri:
+    users:
+      __name: "PEERTUBE_SEARCH_REMOTEURI_USERS"
+      __format: "json"
+    anonymous:
+      __name: "PEERTUBE_SEARCH_REMOTEURI_ANONYMOUS"
+      __format: "json"
+
+import:
+  videos:
+    http:
+      enabled:
+        __name: "PEERTUBE_IMPORT_VIDEOS_HTTP"
+        __format: "json"
+    torrent:
+      enabled:
+        __name: "PEERTUBE_IMPORT_VIDEOS_TORRENT"
+        __format: "json"
+
 transcoding:
   enabled:
     __name: "PEERTUBE_TRANSCODING_ENABLED"