Improve e2e workflow and add doc

10 files changed, 106 insertions, 37 deletions

diff --git a/README.md b/README.md
index 61dc7c9f9..e484dbb9e 100644
--- a/README.md
+++ b/README.md
@@ -31,8 +31,8 @@ Be part of a network of multiple small federated, interoperable video hosting pr
 
   <br />
 
-  <a href="https://automate.browserstack.com/public-build/VHUxYy9zYnZqWnkxTTcyNEpPRVdzY2VzN1VhY3hBQUIrYTk2NGFtMnMvTT0tLWxuMk1vVnBzeDJ4cFpFY1JEK2xjSUE9PQ==--cf445693d1fc03efd86a3a5030d079a0de3ece7a">
-    <img src="https://automate.browserstack.com/badge.svg?badge_key=VHUxYy9zYnZqWnkxTTcyNEpPRVdzY2VzN1VhY3hBQUIrYTk2NGFtMnMvTT0tLWxuMk1vVnBzeDJ4cFpFY1JEK2xjSUE9PQ==--cf445693d1fc03efd86a3a5030d079a0de3ece7a"/>
+  <a href="https://automate.browserstack.com/public-build/ZEZqamJQUXFQd1l3cFp3QmxLSVVwdjBGZjNGc3J2M09INFpka296em9VYz0tLUowWVdoemxkY1hBOU9aZzNlY1htZ3c9PQ==--68e0184ce76481d36559d681d9cddc68235ff536">
+    <img src="https://automate.browserstack.com/badge.svg?badge_key=ZEZqamJQUXFQd1l3cFp3QmxLSVVwdjBGZjNGc3J2M09INFpka296em9VYz0tLUowWVdoemxkY1hBOU9aZzNlY1htZ3c9PQ==--68e0184ce76481d36559d681d9cddc68235ff536"/>
   </a>
 
   <a href="https://weblate.framasoft.org/projects/peertube/angular/">
diff --git a/client/e2e/src/po/player.po.ts b/client/e2e/src/po/player.po.ts
index 9d6e21009..372e8ab20 100644
--- a/client/e2e/src/po/player.po.ts
+++ b/client/e2e/src/po/player.po.ts
@@ -5,13 +5,12 @@ export class PlayerPage {
   getWatchVideoPlayerCurrentTime () {
     const elem = $('video')
 
-    if (isIOS()) {
-      return elem.getAttribute('currentTime')
-        .then(t => parseInt(t, 10))
-        .then(t => Math.round(t))
-    }
+    const p = isIOS()
+      ? elem.getAttribute('currentTime')
+      : elem.getProperty('currentTime')
 
-    return elem.getProperty('currentTime')
+    return p.then(t => parseInt(t + '', 10))
+            .then(t => Math.ceil(t))
   }
 
   waitUntilPlaylistInfo (text: string, maxTime: number) {
@@ -26,7 +25,7 @@ export class PlayerPage {
     })
   }
 
-  async playAndPauseVideo (isAutoplay: boolean) {
+  async playAndPauseVideo (isAutoplay: boolean, waitUntilSec: number) {
     const videojsElem = () => $('div.video-js')
 
     await videojsElem().waitForExist()
@@ -43,10 +42,8 @@ export class PlayerPage {
     await browserSleep(2000)
 
     await browser.waitUntil(async () => {
-      return !await $('.vjs-loading-spinner').isDisplayedInViewport()
-    }, { timeout: 20 * 1000 })
-
-    await browserSleep(4000)
+      return (await this.getWatchVideoPlayerCurrentTime()) >= 2
+    })
 
     await videojsElem().click()
   }
diff --git a/client/e2e/src/po/video-watch.po.ts b/client/e2e/src/po/video-watch.po.ts
index 01061d5d4..c07f4b25f 100644
--- a/client/e2e/src/po/video-watch.po.ts
+++ b/client/e2e/src/po/video-watch.po.ts
@@ -1,3 +1,4 @@
+import { FIXTURE_URLS } from '../urls'
 import { browserSleep, go } from '../utils'
 
 export class VideoWatchPage {
@@ -50,15 +51,11 @@ export class VideoWatchPage {
   }
 
   goOnP2PMediaLoaderEmbed () {
-    return go(
-      'https://peertube2.cpy.re/videos/embed/969bf103-7818-43b5-94a0-de159e13de50'
-    )
+    return go(FIXTURE_URLS.HLS_EMBED)
   }
 
   goOnP2PMediaLoaderPlaylistEmbed () {
-    return go(
-      'https://peertube2.cpy.re/video-playlists/embed/73804a40-da9a-40c2-b1eb-2c6d9eec8f0a'
-    )
+    return go(FIXTURE_URLS.HLS_PLAYLIST_EMBED)
   }
 
   async clickOnVideo (videoName: string) {
diff --git a/client/e2e/src/urls.ts b/client/e2e/src/urls.ts
new file mode 100644
index 000000000..664c65931
--- /dev/null
+++ b/client/e2e/src/urls.ts
@@ -0,0 +1,10 @@
+const FIXTURE_URLS = {
+  WEBTORRENT_VIDEO: 'https://peertube2.cpy.re/w/122d093a-1ede-43bd-bd34-59d2931ffc5e',
+
+  HLS_EMBED: 'https://peertube2.cpy.re/videos/embed/969bf103-7818-43b5-94a0-de159e13de50',
+  HLS_PLAYLIST_EMBED: 'https://peertube2.cpy.re/video-playlists/embed/73804a40-da9a-40c2-b1eb-2c6d9eec8f0a'
+}
+
+export {
+  FIXTURE_URLS
+}
diff --git a/client/e2e/src/videos.e2e-spec.ts b/client/e2e/src/videos.e2e-spec.ts
index 7f7c814e1..e09e8c675 100644
--- a/client/e2e/src/videos.e2e-spec.ts
+++ b/client/e2e/src/videos.e2e-spec.ts
@@ -4,6 +4,7 @@ import { PlayerPage } from './po/player.po'
 import { VideoUpdatePage } from './po/video-update.po'
 import { VideoUploadPage } from './po/video-upload.po'
 import { VideoWatchPage } from './po/video-watch.po'
+import { FIXTURE_URLS } from './urls'
 import { browserSleep, go, isIOS, isMobileDevice, isSafari } from './utils'
 
 function isUploadUnsupported () {
@@ -91,7 +92,7 @@ describe('Videos workflow', () => {
     let videoNameToExcept = videoName
 
     if (isMobileDevice() || isSafari()) {
-      await go('https://peertube2.cpy.re/w/122d093a-1ede-43bd-bd34-59d2931ffc5e')
+      await go(FIXTURE_URLS.WEBTORRENT_VIDEO)
       videoNameToExcept = 'E2E tests'
     } else {
       await videoWatchPage.clickOnVideo(videoName)
@@ -103,21 +104,21 @@ describe('Videos workflow', () => {
   it('Should play the video', async () => {
     videoWatchUrl = await browser.getUrl()
 
-    await playerPage.playAndPauseVideo(true)
+    await playerPage.playAndPauseVideo(true, 2)
     expect(await playerPage.getWatchVideoPlayerCurrentTime()).toBeGreaterThanOrEqual(2)
   })
 
   it('Should watch the associated embed video', async () => {
     await videoWatchPage.goOnAssociatedEmbed()
 
-    await playerPage.playAndPauseVideo(false)
+    await playerPage.playAndPauseVideo(false, 2)
     expect(await playerPage.getWatchVideoPlayerCurrentTime()).toBeGreaterThanOrEqual(2)
   })
 
   it('Should watch the p2p media loader embed video', async () => {
     await videoWatchPage.goOnP2PMediaLoaderEmbed()
 
-    await playerPage.playAndPauseVideo(false)
+    await playerPage.playAndPauseVideo(false, 2)
     expect(await playerPage.getWatchVideoPlayerCurrentTime()).toBeGreaterThanOrEqual(2)
   })
 
diff --git a/client/e2e/wdio.main.conf.ts b/client/e2e/wdio.main.conf.ts
index e2ef99967..7f4c7f7ee 100644
--- a/client/e2e/wdio.main.conf.ts
+++ b/client/e2e/wdio.main.conf.ts
@@ -52,7 +52,7 @@ export const config = {
   //
   // If you only want to run your tests until a specific amount of tests have failed use
   // bail (default is 0 - don't bail, run all tests).
-  bail: 1,
+  bail: 0,
   //
   // Set a base URL in order to shorten url command calls. If your `url` parameter starts
   // with `/`, the base url gets prepended, not including the path portion of your baseUrl.
@@ -79,7 +79,7 @@ export const config = {
   framework: 'mocha',
   //
   // The number of times to retry the entire specfile when it fails as a whole
-  // specFileRetries: 1,
+  specFileRetries: 2,
   //
   // Delay in seconds between the spec file retry attempts
   // specFileRetriesDelay: 0,
diff --git a/package.json b/package.json
index d6efb1db0..4f4f371d8 100644
--- a/package.json
+++ b/package.json
@@ -22,7 +22,7 @@
   },
   "typings": "*.d.ts",
   "scripts": {
-    "e2e": "bash ./scripts/e2e/index.sh",
+    "e2e:browserstack": "bash ./scripts/e2e/browserstack.sh",
     "e2e:local": "bash ./scripts/e2e/local.sh",
     "setup:cli": "bash ./scripts/setup/cli.sh",
     "build": "bash ./scripts/build/index.sh",
diff --git a/scripts/e2e/index.sh b/scripts/e2e/browserstack.sh
index 49d57f1df..69a12d14c 100755
--- a/scripts/e2e/index.sh
+++ b/scripts/e2e/browserstack.sh
@@ -4,11 +4,6 @@ set -eu
 
 npm run clean:server:test
 
-(
-    cd client
-    npm run webpack -- --config webpack/webpack.video-embed.js --mode development
-)
-
 npm run concurrently -- -k -s first \
-    "cd client && npm run ng -- e2e --port 3333" \
+    "cd client/e2e && ../node_modules/.bin/wdio run ./wdio.browserstack.conf.ts" \
     "NODE_ENV=test NODE_APP_INSTANCE=1 NODE_CONFIG='{ \"log\": { \"level\": \"warn\" }, \"signup\": { \"enabled\": false } }' node dist/server"
diff --git a/scripts/test.sh b/scripts/test.sh
index 01b259fe2..2dc79c6ce 100755
--- a/scripts/test.sh
+++ b/scripts/test.sh
@@ -2,11 +2,6 @@
 
 set -eu
 
-npm run build:server
-npm run setup:cli
-
-npm run ci -- lint
-
 npm run ci -- client
 npm run ci -- cli-plugin
 npm run ci -- api-1
@@ -14,3 +9,5 @@ npm run ci -- api-2
 npm run ci -- api-3
 npm run ci -- api-4
 npm run ci -- external-plugins
+
+npm run ci -- lint
diff --git a/support/doc/development/tests.md b/support/doc/development/tests.md
new file mode 100644
index 000000000..e311d3267
--- /dev/null
+++ b/support/doc/development/tests.md
@@ -0,0 +1,72 @@
+# Tests
+
+## Preparation
+
+Prepare PostgreSQL user so PeerTube can delete/create the test databases:
+
+```
+$ sudo -u postgres createuser you_username --createdb --superuser
+```
+
+Prepare databases:
+
+```
+$ npm run clean:server:test
+```
+
+Build PeerTube:
+
+```
+$ npm run build
+```
+
+## Server tests
+
+### Dependencies
+
+Run docker containers needed by some test files:
+
+```
+$ sudo docker run -p 9444:9000 chocobozzz/s3-ninja
+$ sudo docker run -p 10389:10389 chocobozzz/docker-test-openldap
+```
+
+### Test
+
+To run all test suites:
+
+```
+$ npm run test # See scripts/test.sh to run a particular suite
+```
+
+To run a particular test file:
+
+```
+TS_NODE_TRANSPILE_ONLY=true mocha -- --timeout 30000 --exit -r ts-node/register -r tsconfig-paths/register --bail server/tests/api/videos/video-transcoder.ts
+```
+
+### Configuration
+
+Some env variables can be defined to disable/enable some tests:
+
+ * `DISABLE_HTTP_IMPORT_TESTS`: disable import tests (because of youtube that could rate limit your IP)
+ * `ENABLE_OBJECT_STORAGE_TESTS=true`: enable object storage tests (needs a docker container first)
+
+
+## Client E2E tests
+
+### Local tests
+
+To run tests on local web browsers (comment web browsers you don't have in `client/e2e/wdio.local.conf.ts`):
+
+```
+$ npm run e2e:local
+```
+
+### Browserstack tests
+
+To run tests on browser stack:
+
+```
+$ BROWSERSTACK_USER=your_user BROWSERSTACK_KEY=your_key npm run e2e:browserstack
+```