From 7abb5c5da1ed7c6945e81b9e3da0a14f623f69ea Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 30 Jul 2020 14:44:08 +0200 Subject: [PATCH] Use parallel to run tests in parallel --- .github/workflows/test.yml | 2 +- .travis.yml | 58 ------------------------------ scripts/ci.sh | 72 +++++++++++++++++++++++++++++--------- server/tests/api/ci-1.sh | 10 ------ server/tests/api/ci-2.sh | 10 ------ server/tests/api/ci-3.sh | 8 ----- server/tests/api/ci-4.sh | 11 ------ 7 files changed, 57 insertions(+), 114 deletions(-) delete mode 100644 .travis.yml delete mode 100644 server/tests/api/ci-1.sh delete mode 100644 server/tests/api/ci-2.sh delete mode 100644 server/tests/api/ci-3.sh delete mode 100644 server/tests/api/ci-4.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a2f93ce3f..6b9119e2a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: - name: Setup system dependencies run: | - sudo apt-get install postgresql-client-common redis-tools + sudo apt-get install postgresql-client-common redis-tools parallel wget --quiet --no-check-certificate "https://download.cpy.re/ffmpeg/ffmpeg-release-4.0.3-64bit-static.tar.xz" tar xf ffmpeg-release-4.0.3-64bit-static.tar.xz mkdir -p $HOME/bin diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 3d287a782..000000000 --- a/.travis.yml +++ /dev/null @@ -1,58 +0,0 @@ -language: node_js - -node_js: - - "10" - -git: - depth: 1 - -addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-4.9 - postgresql: "9.6" - -cache: - directories: - - $HOME/.cache/yarn - - $HOME/fixtures - -sudo: false - -services: - - postgresql - - redis-server - -install: - - CC=gcc-4.9 CXX=g++-4.9 yarn install - -before_script: - - wget --no-check-certificate "https://download.cpy.re/ffmpeg/ffmpeg-release-4.0.3-64bit-static.tar.xz" - - tar xf ffmpeg-release-4.0.3-64bit-static.tar.xz - - mkdir -p $HOME/bin - - cp ffmpeg-*/{ffmpeg,ffprobe} $HOME/bin - - export PATH=$HOME/bin:$PATH - - psql -c "create user peertube with password 'peertube';" -U postgres - -matrix: - include: - - env: TEST_SUITE=misc - - env: TEST_SUITE=api-1 - - env: TEST_SUITE=api-2 - - env: TEST_SUITE=api-3 - - env: TEST_SUITE=api-4 - - env: TEST_SUITE=cli - - env: TEST_SUITE=lint - -script: - - NODE_PENDING_JOB_WAIT=2000 travis_retry npm run ci -- "$TEST_SUITE" - -after_failure: - - cat test1/logs/peertube.log - - cat test2/logs/peertube.log - - cat test3/logs/peertube.log - - cat test4/logs/peertube.log - - cat test5/logs/peertube.log - - cat test6/logs/peertube.log diff --git a/scripts/ci.sh b/scripts/ci.sh index cef276da3..448c1c91f 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -9,42 +9,82 @@ fi killall -q peertube || true -perl -0777 -i -pe 's#proxy:(\n\s+)enabled: false\n\s+url: ""#proxy:$1enabled: true$1url: "http://188.165.225.149:7899"#' config/test.yaml +retries=3 +jobs=2 + +runTest () { + retries=3 + jobs=$1 + shift + files=$@ + + echo $files + + MOCHA_PARALLEL=true parallel -t -j $jobs --retries $retries \ + npm run mocha -- -c --timeout 30000 --exit --require ts-node/register --require tsconfig-paths/register --bail \ + ::: $files +} + +findTestFiles () { + find $1 -type f -name "*.ts" | grep -v index.ts | xargs echo +} if [ "$1" = "misc" ]; then npm run build -- --light - TS_NODE_FILES=true mocha --timeout 5000 --exit --require ts-node/register --require tsconfig-paths/register --bail \ - server/tests/client.ts \ - server/tests/feeds/index.ts \ - server/tests/misc-endpoints.ts \ - server/tests/helpers/index.ts \ - server/tests/plugins/index.ts + + feedsFiles=$(findTestFiles server/tests/feeds) + helperFiles=$(findTestFiles server/tests/helpers) + pluginsFiles=$(findTestFiles server/tests/plugins) + miscFiles=server/tests/client.ts server/tests/misc-endpoints.ts + + TS_NODE_FILES=true runTest 2 $feedsFiles $helperFiles $pluginsFiles $miscFiles elif [ "$1" = "cli" ]; then npm run build:server npm run setup:cli - mocha --timeout 5000 --exit --require ts-node/register --require tsconfig-paths/register --bail server/tests/cli/index.ts + + cliFiles=$(findTestFiles server/tests/cli) + + runTest 1 $cliFiles elif [ "$1" = "api-1" ]; then npm run build:server - sh ./server/tests/api/ci-1.sh 2 + + checkParamFiles=$(findTestFiles server/tests/api/check-params) + notificationsFiles=$(findTestFiles server/tests/api/notifications) + searchFiles=$(findTestFiles server/tests/api/search) + + runTest 2 $notificationsFiles $searchFiles $checkParamFiles elif [ "$1" = "api-2" ]; then npm run build:server - sh ./server/tests/api/ci-2.sh 2 + + serverFiles=$(findTestFiles server/tests/api/server) + usersFiles=$(findTestFiles server/tests/api/users) + + runTest 2 $serverFiles $usersFiles elif [ "$1" = "api-3" ]; then npm run build:server - sh ./server/tests/api/ci-3.sh 2 + + videosFiles=$(findTestFiles server/tests/api/videos) + + runTest 1 $videosFiles elif [ "$1" = "api-4" ]; then npm run build:server - sh ./server/tests/api/ci-4.sh 2 + + activitypubFiles=$(findTestFiles server/tests/api/moderation) + redundancyFiles=$(findTestFiles server/tests/api/redundancy) + activitypubFiles=$(findTestFiles server/tests/api/activitypub) + + TS_NODE_FILES=true runTest 2 $activitypubFiles $redundancyFiles $activitypubFiles elif [ "$1" = "external-plugins" ]; then npm run build:server - mocha --timeout 5000 --exit --require ts-node/register --require tsconfig-paths/register --bail server/tests/external-plugins/index.ts + + externalPluginsFiles=$(findTestFiles server/tests/external-plugins) + + runTest 1 $externalPluginsFiles elif [ "$1" = "lint" ]; then - npm run eslint -- --ext .ts "server/**/*.ts" "shared/**/*.ts" + npm run eslint -- --ext .ts "server/**/*.ts" "shared/**/*.ts" "scripts/**/*.ts" npm run swagger-cli -- validate support/doc/api/openapi.yaml ( cd client npm run lint ) fi - -git checkout -- config/test.yaml diff --git a/server/tests/api/ci-1.sh b/server/tests/api/ci-1.sh deleted file mode 100644 index faa440785..000000000 --- a/server/tests/api/ci-1.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env sh - -set -eu - -checkParamFiles=$(find server/tests/api/check-params -type f | grep -v index.ts | xargs echo) -notificationsFiles=$(find server/tests/api/notifications -type f | grep -v index.ts | xargs echo) -searchFiles=$(find server/tests/api/search -type f | grep -v index.ts | xargs echo) - -MOCHA_PARALLEL=true npm run mocha -- --timeout 30000 --exit --require ts-node/register --require tsconfig-paths/register --bail \ - $notificationsFiles $searchFiles $checkParamFiles diff --git a/server/tests/api/ci-2.sh b/server/tests/api/ci-2.sh deleted file mode 100644 index 016df1cfc..000000000 --- a/server/tests/api/ci-2.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env sh - -set -eu - -serverFiles=$(find server/tests/api/server -type f | grep -v index.ts | xargs echo) -usersFiles=$(find server/tests/api/users -type f | grep -v index.ts | xargs echo) - -MOCHA_PARALLEL=true npm run mocha -- --parallel --jobs $1 --timeout 30000 --exit \ - --require ts-node/register --require tsconfig-paths/register --bail \ - $serverFiles $usersFiles diff --git a/server/tests/api/ci-3.sh b/server/tests/api/ci-3.sh deleted file mode 100644 index ae5bb735a..000000000 --- a/server/tests/api/ci-3.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env sh - -set -eu - -videosFiles=$(find server/tests/api/videos -type f | grep -v index.ts | xargs echo) - -npm run mocha -- --timeout 30000 --exit --require ts-node/register --require tsconfig-paths/register --bail \ - $videosFiles diff --git a/server/tests/api/ci-4.sh b/server/tests/api/ci-4.sh deleted file mode 100644 index 4998de364..000000000 --- a/server/tests/api/ci-4.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env sh - -set -eu - -activitypubFiles=$(find server/tests/api/moderation -type f | grep -v index.ts | xargs echo) -redundancyFiles=$(find server/tests/api/redundancy -type f | grep -v index.ts | xargs echo) -activitypubFiles=$(find server/tests/api/activitypub -type f | grep -v index.ts | xargs echo) - -TS_NODE_FILES=true MOCHA_PARALLEL=true npm run mocha -- --parallel --jobs $1 --timeout 30000 --exit \ - --require ts-node/register --require tsconfig-paths/register --bail \ - $redundancyFiles $activitypubFiles -- 2.41.0