Merge branch 'develop' into pr/1217

16 files changed, 157 insertions, 44 deletions

diff --git a/scripts/build/client.sh b/scripts/build/client.sh
index 62daf98cf..be3eef802 100755
--- a/scripts/build/client.sh
+++ b/scripts/build/client.sh
@@ -41,7 +41,7 @@ if [ -z ${1+x} ] || [ "$1" != "--light" ]; then
         languages=("fr_FR")
     else
         # Supported languages
-        languages=("fr_FR" "pt_BR" "sv_SE" "eu_ES" "ca_ES" "cs_CZ" "eo" "zh_Hant_TW" "de_DE" "es_ES" "oc" "zh_Hans_CN")
+        languages=("pl_PL" "it_IT" "ru_RU" "fr_FR" "pt_BR" "sv_SE" "eu_ES" "ca_ES" "cs_CZ" "eo" "zh_Hant_TW" "de_DE" "es_ES" "oc" "zh_Hans_CN")
     fi
 
     for lang in "${languages[@]}"; do
diff --git a/scripts/clean/server/test.sh b/scripts/clean/server/test.sh
index 235ff52cc..b897c30ba 100755
--- a/scripts/clean/server/test.sh
+++ b/scripts/clean/server/test.sh
@@ -18,6 +18,7 @@ removeFiles () {
 
 dropRedis () {
   redis-cli KEYS "bull-localhost:900$1*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
+  redis-cli KEYS "redis-localhost:900$1*" | grep -v empty | xargs --no-run-if-empty redis-cli DEL
 }
 
 for i in $(seq 1 6); do
diff --git a/scripts/dev/server.sh b/scripts/dev/server.sh
index 9b8fddac6..b4675c57f 100755
--- a/scripts/dev/server.sh
+++ b/scripts/dev/server.sh
@@ -4,7 +4,7 @@ set -eu
 
 if [ ! -f "./client/dist/en_US/index.html" ]; then
   echo "client/dist/en_US/index.html does not exist, compile client files..."
-  npm run build:client
+  npm run build:client -- --light
 fi
 
 npm run watch:server
diff --git a/scripts/generate-api-doc.sh b/scripts/generate-api-doc.sh
deleted file mode 100755
index 7d48db7a5..000000000
--- a/scripts/generate-api-doc.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-
-set -eu
-
-npm run spectacle-docs -- -t support/doc/api/html support/doc/api/openapi.yaml
diff --git a/scripts/generate-cli-doc.sh b/scripts/generate-cli-doc.sh
new file mode 100755
index 000000000..0d00f183b
--- /dev/null
+++ b/scripts/generate-cli-doc.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+set -eu
+
+node_modules/marked-man/bin/marked-man server/tools/README.md > dist/server/tools/peertube.8
diff --git a/scripts/generate-code-contributors.ts b/scripts/generate-code-contributors.ts
index 3d37372d1..96110307a 100755
--- a/scripts/generate-code-contributors.ts
+++ b/scripts/generate-code-contributors.ts
@@ -41,7 +41,7 @@ async function run () {
 }
 
 function get (url: string, headers: any = {}) {
-  return doRequest({
+  return doRequest<any>({
     uri: url,
     json: true,
     headers: Object.assign(headers, {
@@ -70,11 +70,17 @@ async function fetchGithub (url: string) {
 
 async function fetchZanata (zanataUsername: string, zanataPassword: string) {
   const today = new Date().toISOString().split('T')[0]
-  const url = `https://trad.framasoft.org/zanata/rest/project/peertube/version/develop/contributors/2018-01-01..${today}`
+  const year2018 = `https://trad.framasoft.org/zanata/rest/project/peertube/version/develop/contributors/2018-01-01..2019-01-01`
+  const year2019 = `https://trad.framasoft.org/zanata/rest/project/peertube/version/develop/contributors/2019-01-01..${today}`
 
   const headers = {
     'X-Auth-User': zanataUsername,
     'X-Auth-Token': zanataPassword
   }
-  return get(url, headers)
+  const [ results2018, results2019 ] = await Promise.all([
+    get(year2018, headers),
+    get(year2019, headers)
+  ])
+
+  return results2018.concat(results2019)
 }
diff --git a/scripts/help.sh b/scripts/help.sh
index 8ac090139..bc38bdb40 100755
--- a/scripts/help.sh
+++ b/scripts/help.sh
@@ -18,6 +18,7 @@ printf "  reset-password -- -u [user] -> Reset the password of user [user]\n"
 printf "  create-transcoding-job -- -v [video UUID] \n"
 printf "                              -> Create a transcoding job for a particular video\n"
 printf "  prune-storage               -> Delete (after confirmation) unknown video files/thumbnails/previews... (due to a bad video deletion, transcoding job not finished...)\n"
+printf "  optimize-old-videos         -> Re-transcode videos that have a high bitrate, to make them suitable for streaming over slow connections"
 printf "  dev                         -> Watch, run the livereload and run the server so that you can develop the application\n"
 printf "  start                       -> Run the server\n"
 printf "  update-host                 -> Upgrade scheme/host in torrent files according to the webserver configuration (config/ folder)\n"
diff --git a/scripts/i18n/create-custom-files.ts b/scripts/i18n/create-custom-files.ts
index eed3182a6..664207e1c 100755
--- a/scripts/i18n/create-custom-files.ts
+++ b/scripts/i18n/create-custom-files.ts
@@ -23,12 +23,15 @@ const playerKeys = {
   'Speed': 'Speed',
   'Subtitles/CC': 'Subtitles/CC',
   'peers': 'peers',
+  'peer': 'peer',
   'Go to the video page': 'Go to the video page',
   'Settings': 'Settings',
   'Uses P2P, others may know you are watching this video.': 'Uses P2P, others may know you are watching this video.',
   'Copy the video URL': 'Copy the video URL',
   'Copy the video URL at the current time': 'Copy the video URL at the current time',
-  'Copy embed code': 'Copy embed code'
+  'Copy embed code': 'Copy embed code',
+  'Total downloaded: ': 'Total downloaded: ',
+  'Total uploaded: ': 'Total uploaded: '
 }
 const playerTranslations = {
   target: join(__dirname, '../../../client/src/locale/source/player_en_US.xml'),
@@ -42,6 +45,12 @@ values(VIDEO_CATEGORIES)
   .concat(values(VIDEO_PRIVACIES))
   .concat(values(VIDEO_STATES))
   .concat(values(VIDEO_IMPORT_STATES))
+  .concat([
+    'This video does not exist.',
+    'We cannot fetch the video. Please try again later.',
+    'Sorry',
+    'This video is not available because the remote instance is not responding.'
+  ])
   .forEach(v => serverKeys[v] = v)
 
 // More keys
@@ -103,4 +112,4 @@ function saveToXliffFile (jsonTranslations: TranslationType, cb: Function) {
 function handleError (err: any) {
   console.error(err)
   process.exit(-1)
-}
\ No newline at end of file
+}
diff --git a/scripts/i18n/pull-hook.sh b/scripts/i18n/pull-hook.sh
index 3a8394110..b0668436d 100755
--- a/scripts/i18n/pull-hook.sh
+++ b/scripts/i18n/pull-hook.sh
@@ -7,6 +7,7 @@ set -eu
 
 for i in 1 2 3; do
     perl -pi -e 's|&lt;x id=(.+?)/&gt;([^"])|<x id=\1/>\2|g' client/src/locale/target/*.xml
+    perl -0pi -e 's|<source>(.+?)</source>\s*<context-group |<source>\1</source><target>\1</target><context-group |g' client/src/locale/target/angular_*.xml
 done
 
 npm run i18n:xliff2json
diff --git a/scripts/openapi-peertube-version.sh b/scripts/openapi-peertube-version.sh
new file mode 100755
index 000000000..4eb481e64
--- /dev/null
+++ b/scripts/openapi-peertube-version.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# Version key/value should be on his own line
+PACKAGE_VERSION=$(node -p "require('./package.json').version")
+
+sed -i "s/\(^\s*\)version: .*/\1version: $PACKAGE_VERSION/" ./support/doc/api/openapi.yaml
diff --git a/scripts/optimize-old-videos.ts b/scripts/optimize-old-videos.ts
new file mode 100644
index 000000000..1bee1b0f3
--- /dev/null
+++ b/scripts/optimize-old-videos.ts
@@ -0,0 +1,67 @@
+import { CONFIG, VIDEO_TRANSCODING_FPS } from '../server/initializers/constants'
+import { getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution, getDurationFromVideoFile } from '../server/helpers/ffmpeg-utils'
+import { getMaxBitrate } from '../shared/models/videos'
+import { VideoModel } from '../server/models/video/video'
+import { optimizeVideofile } from '../server/lib/video-transcoding'
+import { initDatabaseModels } from '../server/initializers'
+import { join, basename, dirname } from 'path'
+import { copy, remove, move } from 'fs-extra'
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+let currentVideoId = null
+let currentFile = null
+
+process.on('SIGINT', async function () {
+  console.log('Cleaning up temp files')
+  await remove(`${currentFile}_backup`)
+  await remove(`${dirname(currentFile)}/${currentVideoId}-transcoded.mp4`)
+  process.exit(0)
+})
+
+async function run () {
+  await initDatabaseModels(true)
+
+  const localVideos = await VideoModel.listLocal()
+
+  for (const video of localVideos) {
+    currentVideoId = video.id
+    for (const file of video.VideoFiles) {
+      currentFile = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file))
+
+      const [ videoBitrate, fps, resolution ] = await Promise.all([
+        getVideoFileBitrate(currentFile),
+        getVideoFileFPS(currentFile),
+        getVideoFileResolution(currentFile)
+      ])
+
+      const maxBitrate = getMaxBitrate(resolution.videoFileResolution, fps, VIDEO_TRANSCODING_FPS)
+      const isMaxBitrateExceeded = videoBitrate > maxBitrate
+      if (isMaxBitrateExceeded) {
+        console.log('Optimizing video file %s with bitrate %s kbps (max: %s kbps)',
+          basename(currentFile), videoBitrate / 1000, maxBitrate / 1000)
+        const backupFile = `${currentFile}_backup`
+        await copy(currentFile, backupFile)
+        await optimizeVideofile(video, file)
+        const originalDuration = await getDurationFromVideoFile(backupFile)
+        const newDuration = await getDurationFromVideoFile(currentFile)
+        if (originalDuration === newDuration) {
+          console.log('Finished optimizing %s', basename(currentFile))
+          await remove(backupFile)
+        } else {
+          console.log('Failed to optimize %s, restoring original', basename(currentFile))
+          move(backupFile, currentFile, { overwrite: true })
+          await video.createTorrentAndSetInfoHash(file)
+          await file.save()
+        }
+      }
+    }
+  }
+
+  console.log('Finished optimizing videos')
+}
diff --git a/scripts/prune-storage.ts b/scripts/prune-storage.ts
index 4088fa700..c9e4dbd4b 100755
--- a/scripts/prune-storage.ts
+++ b/scripts/prune-storage.ts
@@ -5,6 +5,7 @@ import { VideoModel } from '../server/models/video/video'
 import { initDatabaseModels } from '../server/initializers'
 import { remove, readdir } from 'fs-extra'
 import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy'
+import { getUUIDFromFilename } from '../server/helpers/utils'
 
 run()
   .then(() => process.exit(0))
@@ -18,7 +19,8 @@ async function run () {
 
   const storageOnlyOwnedToPrune = [
     CONFIG.STORAGE.VIDEOS_DIR,
-    CONFIG.STORAGE.TORRENTS_DIR
+    CONFIG.STORAGE.TORRENTS_DIR,
+    CONFIG.STORAGE.REDUNDANCY_DIR
   ]
 
   const storageForAllToPrune = [
@@ -35,6 +37,9 @@ async function run () {
     toDelete = toDelete.concat(await pruneDirectory(directory, false))
   }
 
+  const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
+  toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)))
+
   if (toDelete.length === 0) {
     console.log('No files to delete.')
     return
@@ -82,15 +87,6 @@ async function pruneDirectory (directory: string, onlyOwned = false) {
   return toDelete
 }
 
-function getUUIDFromFilename (filename: string) {
-  const regex = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/
-  const result = filename.match(regex)
-
-  if (!result || Array.isArray(result) === false) return null
-
-  return result[0]
-}
-
 async function askConfirmation () {
   return new Promise((res, rej) => {
     prompt.start()
@@ -99,6 +95,7 @@ async function askConfirmation () {
         confirm: {
           type: 'string',
           description: 'These following unused files can be deleted, but please check your backups first (bugs happen).' +
+            ' Notice PeerTube must have been stopped when your ran this script.' +
             ' Can we delete these files?',
           default: 'n',
           required: true
diff --git a/scripts/release.sh b/scripts/release.sh
index 7b577ef35..08061fe6f 100755
--- a/scripts/release.sh
+++ b/scripts/release.sh
@@ -27,15 +27,15 @@ fi
 maintainer_public_key=${MAINTAINER_GPG:-"583A612D890159BE"}
 
 branch=$(git symbolic-ref --short -q HEAD)
-if [ "$branch" != "develop" ] && [[ "$branch" != feature/* ]]; then
+if [ "$branch" != "develop" ] && [[ "$branch" != release/* ]]; then
   echo "Need to be on develop or release branch."
   exit -1
 fi
 
 version="v$1"
 github_prerelease_option=""
-if [[ "$version" = *".pre."* ]]; then
-  echo "This is a pre-release."
+if [[ "$version" = *"-alpha."* ]] || [[ "$version" = *"-beta."* ]] || [[ "$version" = *"-rc."* ]]; then
+  echo -e "This is a pre-release.\n"
   github_prerelease_option="--pre-release"
 fi
 
@@ -43,9 +43,9 @@ directory_name="peertube-$version"
 zip_name="peertube-$version.zip"
 tar_name="peertube-$version.tar.xz"
 
-changelog=$(awk -v version="$version" '/## v/ { printit = $2 == version }; printit;' CHANGELOG.md | grep -v "$version" | sed '1{/^$/d}')
+changelog=$(awk -v version="$version" '/## v/ { printit = $2 == version }; printit;' CHANGELOG.md | grep -v "## $version" | sed '1{/^$/d}')
 
-printf "Changelog will be:\\n%s\\n" "$changelog"
+printf "Changelog will be:\\n\\n%s\\n\\n" "$changelog"
 
 read -p "Are you sure to release? " -n 1 -r
 echo
@@ -60,7 +60,9 @@ fi
 
 npm version -f --no-git-tag-version --no-commit-hooks "$1"
 
-git commit package.json client/package.json -m "Bumped to version $version"
+./scripts/openapi-peertube-version.sh
+
+git commit package.json client/package.json ./support/doc/api/openapi.yaml -m "Bumped to version $version"
 git tag -s -a "$version" -m "$version"
 
 npm run build
diff --git a/scripts/travis.sh b/scripts/travis.sh
index 5d195f902..509b40d87 100755
--- a/scripts/travis.sh
+++ b/scripts/travis.sh
@@ -11,23 +11,25 @@ killall -q peertube || true
 
 if [ "$1" = "misc" ]; then
     npm run build -- --light-fr
-    mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/client.ts server/tests/activitypub.ts \
-        server/tests/feeds/index.ts server/tests/misc-endpoints.ts
-elif [ "$1" = "api" ]; then
-    npm run build:server
-    mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index.ts
+    mocha --timeout 5000 --exit --require ts-node/register --bail server/tests/client.ts \
+        server/tests/feeds/index.ts \
+        server/tests/misc-endpoints.ts \
+        server/tests/helpers/index.ts
 elif [ "$1" = "cli" ]; then
     npm run build:server
-    mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/cli/index.ts
+    mocha --timeout 5000 --exit --require ts-node/register --bail server/tests/cli/index.ts
 elif [ "$1" = "api-1" ]; then
     npm run build:server
-    mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-1.ts
+    mocha --timeout 5000 --exit --require ts-node/register --bail server/tests/api/index-1.ts
 elif [ "$1" = "api-2" ]; then
     npm run build:server
-    mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-2.ts
+    mocha --timeout 5000 --exit --require ts-node/register --bail server/tests/api/index-2.ts
 elif [ "$1" = "api-3" ]; then
     npm run build:server
-    mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index-3.ts
+    mocha --timeout 5000 --exit --require ts-node/register --bail server/tests/api/index-3.ts
+elif [ "$1" = "api-4" ]; then
+    npm run build:server
+    mocha --timeout 5000 --exit --require ts-node/register --bail server/tests/api/index-4.ts
 elif [ "$1" = "lint" ]; then
     npm run tslint -- --project ./tsconfig.json -c ./tslint.json server.ts "server/**/*.ts" "shared/**/*.ts"
 
diff --git a/scripts/update-host.ts b/scripts/update-host.ts
index 1dc19664d..64eba867a 100755
--- a/scripts/update-host.ts
+++ b/scripts/update-host.ts
@@ -4,7 +4,7 @@ import { VideoModel } from '../server/models/video/video'
 import { ActorModel } from '../server/models/activitypub/actor'
 import {
   getAccountActivityPubUrl,
-  getAnnounceActivityPubUrl,
+  getVideoAnnounceActivityPubUrl,
   getVideoActivityPubUrl, getVideoChannelActivityPubUrl,
   getVideoCommentActivityPubUrl
 } from '../server/lib/activitypub'
@@ -13,6 +13,7 @@ import { VideoCommentModel } from '../server/models/video/video-comment'
 import { getServerActor } from '../server/helpers/utils'
 import { AccountModel } from '../server/models/account/account'
 import { VideoChannelModel } from '../server/models/video/video-channel'
+import { VideoStreamingPlaylistModel } from '../server/models/video/video-streaming-playlist'
 
 run()
   .then(() => process.exit(0))
@@ -78,7 +79,7 @@ async function run () {
 
     console.log('Updating video share ' + videoShare.url)
 
-    videoShare.url = getAnnounceActivityPubUrl(videoShare.Video.url, videoShare.Actor)
+    videoShare.url = getVideoAnnounceActivityPubUrl(videoShare.Actor, videoShare.Video)
     await videoShare.save()
   }
 
@@ -109,11 +110,9 @@ async function run () {
 
   console.log('Updating video and torrent files.')
 
-  const videos = await VideoModel.list()
+  const videos = await VideoModel.listLocal()
   for (const video of videos) {
-    if (video.isOwned() === false) continue
-
-    console.log('Updated video ' + video.uuid)
+    console.log('Updating video ' + video.uuid)
 
     video.url = getVideoActivityPubUrl(video)
     await video.save()
@@ -122,5 +121,12 @@ async function run () {
       console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid)
       await video.createTorrentAndSetInfoHash(file)
     }
+
+    for (const playlist of video.VideoStreamingPlaylists) {
+      playlist.playlistUrl = CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsMasterPlaylistStaticPath(video.uuid)
+      playlist.segmentsSha256Url = CONFIG.WEBSERVER.URL + VideoStreamingPlaylistModel.getHlsSha256SegmentsStaticPath(video.uuid)
+
+      await playlist.save()
+    }
   }
 }
diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh
index b29615fb1..4f7c58edd 100755
--- a/scripts/upgrade.sh
+++ b/scripts/upgrade.sh
@@ -20,11 +20,26 @@ if [ ! -e "$PEERTUBE_PATH/versions" -o ! -e "$PEERTUBE_PATH/config/production.ya
   exit 1
 fi
 
+if [ -x "$(command -v awk)" ] && [ -x "$(command -v sed)" ] ; then
+    REMAINING=$(df -k $PEERTUBE_PATH | awk '{ print $4}' | sed -n 2p)
+    ONE_GB=$((1024 * 1024))
+    if [ "$REMAINING" -lt "$ONE_GB" ]; then
+    echo "Error - not enough free space for upgrading"
+    echo ""
+    echo "Make sure you have at least 1 GB of free space in $PEERTUBE_PATH"
+    exit 1
+    fi
+fi
 
 # Backup database
 SQL_BACKUP_PATH="$PEERTUBE_PATH/backup/sql-peertube_prod-$(date +"%Y%m%d-%H%M").bak" 
+DB_USER=$(node -e "console.log(require('js-yaml').safeLoad(fs.readFileSync('$PEERTUBE_PATH/config/production.yaml', 'utf8'))['database']['username'])")
+DB_PASS=$(node -e "console.log(require('js-yaml').safeLoad(fs.readFileSync('$PEERTUBE_PATH/config/production.yaml', 'utf8'))['database']['password'])")
+DB_HOST=$(node -e "console.log(require('js-yaml').safeLoad(fs.readFileSync('$PEERTUBE_PATH/config/production.yaml', 'utf8'))['database']['hostname'])")
+DB_SUFFIX=$(node -e "console.log(require('js-yaml').safeLoad(fs.readFileSync('$PEERTUBE_PATH/config/production.yaml', 'utf8'))['database']['suffix'])")
 mkdir -p $PEERTUBE_PATH/backup
-pg_dump -U peertube -W -h localhost -F c peertube_prod -f "$SQL_BACKUP_PATH" 
+
+PGPASSWORD=$DB_PASS pg_dump -U $DB_USER -h $DB_HOST -F c "peertube${DB_SUFFIX}" -f "$SQL_BACKUP_PATH"
 
 # If there is a pre-release, give the user a choice which one to install.
 RELEASE_VERSION=$(curl -s https://api.github.com/repos/chocobozzz/peertube/releases/latest | grep tag_name | cut -d '"' -f 4)