Build nightly using github action

10 files changed, 191 insertions, 162 deletions

diff --git a/.github/actions/reusable-deploy/action.yml b/.github/actions/reusable-deploy/action.yml
new file mode 100644
index 000000000..bc69a2e43
--- /dev/null
+++ b/.github/actions/reusable-deploy/action.yml
@@ -0,0 +1,46 @@
+name: "Reusable deploy on builds.joinpeertube.org"
+
+description: "Reusable deploy on builds.joinpeertube.org"
+
+inputs:
+  source:
+    required: true
+    description: "Source file/files/directory/directories to deploy"
+  destination:
+    required: true
+    description: "Destination directory on builds.joinpeertube.org"
+  knownHosts:
+    required: true
+    description: "Known hosts"
+  deployKey:
+    required: true
+    description: "Deploy key"
+  deployUser:
+    required: true
+    description: "Deploy user"
+  deployHost:
+    required: true
+    description: "Deploy host"
+
+
+runs:
+  using: "composite"
+
+  steps:
+    - name: "Deploy"
+      shell: bash
+      run: |
+        mkdir -p ~/.ssh
+        chmod 700  ~/.ssh
+
+        echo "Adding ssh key to known hosts"
+        echo -e "${{ inputs.knownHosts }}" > ~/.ssh/known_hosts;
+
+        eval `ssh-agent -s`
+
+        echo "Adding ssh deploy key"
+        ssh-add <(echo "${{ inputs.deployKey }}");
+
+        echo "Uploading files"
+
+        scp ${{ inputs.source }} ${{ inputs.deployUser }}@${{ inputs.deployHost }}:../../web/${{ inputs.destination }};
diff --git a/.github/actions/reusable-prepare-peertube-build/action.yml b/.github/actions/reusable-prepare-peertube-build/action.yml
new file mode 100644
index 000000000..41ebf71c5
--- /dev/null
+++ b/.github/actions/reusable-prepare-peertube-build/action.yml
@@ -0,0 +1,31 @@
+name: "Reusable prepare PeerTube build"
+
+description: "Reusable prepare PeerTube build"
+
+inputs:
+  node-version:
+    required: true
+    description: 'NodeJS version'
+
+runs:
+  using: "composite"
+
+  steps:
+    - name: Use Node.js
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ inputs.node-version }}
+
+    - name: Cache Node.js modules
+      uses: actions/cache@v2
+      with:
+        path: |
+          **/node_modules
+        key: ${{ runner.OS }}-node-${{ hashFiles('**/yarn.lock') }}
+        restore-keys: |
+          ${{ runner.OS }}-node-
+          ${{ runner.OS }}-
+
+    - name: Install dependencies
+      shell: bash
+      run: yarn install --frozen-lockfile
diff --git a/.github/actions/reusable-prepare-peertube-run/action.yml b/.github/actions/reusable-prepare-peertube-run/action.yml
new file mode 100644
index 000000000..1a6cd2cfd
--- /dev/null
+++ b/.github/actions/reusable-prepare-peertube-run/action.yml
@@ -0,0 +1,16 @@
+name: "Reusable prepare PeerTube run"
+description: "Reusable prepare PeerTube run"
+
+runs:
+  using: "composite"
+
+  steps:
+    - name: Setup system dependencies
+      shell: bash
+      run: |
+        sudo apt-get install postgresql-client-common redis-tools parallel
+        wget --quiet --no-check-certificate "https://download.cpy.re/ffmpeg/ffmpeg-release-4.3.1-64bit-static.tar.xz"
+        tar xf ffmpeg-release-4.3.1-64bit-static.tar.xz
+        mkdir -p $HOME/bin
+        cp ffmpeg-*/{ffmpeg,ffprobe} $HOME/bin
+        echo "$HOME/bin" >> $GITHUB_PATH
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 86f675432..f02b88a42 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -29,48 +29,15 @@ jobs:
     env:
       PGUSER: peertube
       PGHOST: localhost
-      NODE_PENDING_JOB_WAIT: 500
 
     steps:
       - uses: actions/checkout@v2
 
-      - name: Use Node.js
-        uses: actions/setup-node@v1
+      - uses: './.github/actions/reusable-prepare-peertube-build'
         with:
           node-version: '12.x'
 
-      - name: Setup system dependencies
-        run: |
-          sudo apt-get install postgresql-client-common redis-tools parallel
-          wget --quiet --no-check-certificate "https://download.cpy.re/ffmpeg/ffmpeg-release-4.3.1-64bit-static.tar.xz"
-          tar xf ffmpeg-release-4.3.1-64bit-static.tar.xz
-          mkdir -p $HOME/bin
-          cp ffmpeg-*/{ffmpeg,ffprobe} $HOME/bin
-          echo "$HOME/bin" >> $GITHUB_PATH
-
-      - name: Cache Node.js modules
-        uses: actions/cache@v2
-        with:
-          path: |
-            **/node_modules
-          key: ${{ runner.OS }}-node-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.OS }}-node-
-            ${{ runner.OS }}-
-
-      - name: Cache fixtures
-        uses: actions/cache@v2
-        with:
-          path: |
-            fixtures
-          key: ${{ runner.OS }}-fixtures-${{ matrix.test_suite }}-${{ hashFiles('fixtures/*') }}
-          restore-keys: |
-            ${{ runner.OS }}-fixtures-${{ matrix.test_suite }}-
-            ${{ runner.OS }}-fixtures-
-            ${{ runner.OS }}-
-
-      - name: Install dependencies
-        run: yarn install --frozen-lockfile
+      - uses: './.github/actions/reusable-prepare-peertube-run'
 
       - name: Build
         run: |
@@ -111,27 +78,11 @@ jobs:
           cat benchmark.json build-time.json startup-time.json
 
       - name: Upload benchmark result
-        env:
-          STATS_DEPLOYEMENT_KNOWN_HOSTS: ${{ secrets.STATS_DEPLOYEMENT_KNOWN_HOSTS }}
-          STATS_DEPLOYEMENT_KEY: ${{ secrets.STATS_DEPLOYEMENT_KEY }}
-          STATS_DEPLOYEMENT_USER: ${{ secrets.STATS_DEPLOYEMENT_USER }}
-          STATS_DEPLOYEMENT_HOST: ${{ secrets.STATS_DEPLOYEMENT_HOST }}
-        run: |
-          mkdir -p ~/.ssh
-          chmod 700  ~/.ssh
-          if [ ! -z ${STATS_DEPLOYEMENT_KNOWN_HOSTS+x} ]; then
-            echo "Adding ssh key to known hosts"
-            echo -e "${STATS_DEPLOYEMENT_KNOWN_HOSTS}" > ~/.ssh/known_hosts;
-          fi
-
-          eval `ssh-agent -s`
-
-          if [ ! -z ${STATS_DEPLOYEMENT_KEY+x} ]; then
-            echo "Adding ssh reployement key"
-            ssh-add <(echo "${STATS_DEPLOYEMENT_KEY}");
-          fi
-
-          if [ ! -z ${STATS_DEPLOYEMENT_KEY+x} ]; then
-            echo "Uploading files"
-            scp benchmark.json build-time.json startup-time.json ${STATS_DEPLOYEMENT_USER}@${STATS_DEPLOYEMENT_HOST}:../../web/peertube-stats;
-          fi
+        uses: './.github/actions/reusable-deploy.yml'
+        with:
+          source: benchmark.json build-time.json startup-time.json
+          destination: peertube-stats
+          knownHosts: ${{ secrets.STATS_DEPLOYEMENT_KNOWN_HOSTS }}
+          deployKey: ${{ secrets.STATS_DEPLOYEMENT_KEY }}
+          deployUser: ${{ secrets.STATS_DEPLOYEMENT_USER }}
+          deployHost: ${{ secrets.STATS_DEPLOYEMENT_HOST }}
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
index 20803bd86..5c0c93886 100644
--- a/.github/workflows/codeql.yml
+++ b/.github/workflows/codeql.yml
@@ -13,7 +13,7 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ next ]
+    branches: [ develop, next ]
   pull_request:
     # The branches below must be a subset of the branches above
     branches: [ next ]
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
new file mode 100644
index 000000000..156b7143a
--- /dev/null
+++ b/.github/workflows/nightly.yml
@@ -0,0 +1,36 @@
+name: Nightly
+
+on:
+  push:
+    branches:
+      - 'next'
+  schedule:
+    - cron: '0 3 * * *'
+
+jobs:
+
+  nightly:
+    runs-on: ubuntu-latest
+
+    steps:
+      -
+        name: Checkout develop
+        uses: actions/checkout@v2
+        with:
+          ref: next
+
+      - uses: './.github/actions/reusable-prepare-peertube-build'
+        with:
+          node-version: '14.x'
+
+      - name: Build
+        run: npm run nightly
+
+      - uses: './.github/actions/reusable-deploy'
+        with:
+          source: ./peertube-nightly-*
+          destination: nightly-test
+          knownHosts: ${{ secrets.STATS_DEPLOYEMENT_KNOWN_HOSTS }}
+          deployKey: ${{ secrets.STATS_DEPLOYEMENT_KEY }}
+          deployUser: ${{ secrets.STATS_DEPLOYEMENT_USER }}
+          deployHost: ${{ secrets.STATS_DEPLOYEMENT_HOST }}
diff --git a/.github/workflows/stats.yml b/.github/workflows/stats.yml
index e50c14720..c87e6fb77 100644
--- a/.github/workflows/stats.yml
+++ b/.github/workflows/stats.yml
@@ -20,24 +20,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Use Node.js
-        uses: actions/setup-node@v1
+      - uses: './.github/actions/reusable-prepare-peertube-build'
         with:
           node-version: '14.x'
 
-      - name: Cache Node.js modules
-        uses: actions/cache@v2
-        with:
-          path: |
-            **/node_modules
-          key: ${{ runner.OS }}-node-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.OS }}-node-
-            ${{ runner.OS }}-
-
-      - name: Install dependencies
-        run: yarn install --frozen-lockfile
-
       - name: Angular bundlewatch
         uses: jackyef/bundlewatch-gh-action@master
         with:
@@ -73,27 +59,11 @@ jobs:
 
       - name: Upload stats
         if: github.event_name != 'pull_request'
-        env:
-          STATS_DEPLOYEMENT_KNOWN_HOSTS: ${{ secrets.STATS_DEPLOYEMENT_KNOWN_HOSTS }}
-          STATS_DEPLOYEMENT_KEY: ${{ secrets.STATS_DEPLOYEMENT_KEY }}
-          STATS_DEPLOYEMENT_USER: ${{ secrets.STATS_DEPLOYEMENT_USER }}
-          STATS_DEPLOYEMENT_HOST: ${{ secrets.STATS_DEPLOYEMENT_HOST }}
-        run: |
-          mkdir -p ~/.ssh
-          chmod 700  ~/.ssh
-          if [ ! -z ${STATS_DEPLOYEMENT_KNOWN_HOSTS+x} ]; then
-            echo "Adding ssh key to known hosts"
-            echo -e "${STATS_DEPLOYEMENT_KNOWN_HOSTS}" > ~/.ssh/known_hosts;
-          fi
-
-          eval `ssh-agent -s`
-
-          if [ ! -z ${STATS_DEPLOYEMENT_KEY+x} ]; then
-            echo "Adding ssh reployement key"
-            ssh-add <(echo "${STATS_DEPLOYEMENT_KEY}");
-          fi
-
-          if [ ! -z ${STATS_DEPLOYEMENT_KEY+x} ]; then
-            echo "Uploading files"
-            scp lighthouse.json client-build-stats.json scc.json ${STATS_DEPLOYEMENT_USER}@${STATS_DEPLOYEMENT_HOST}:../../web/peertube-stats;
-          fi
+        uses: './.github/actions/reusable-deploy'
+        with:
+          source: lighthouse.json client-build-stats.json scc.json
+          destination: peertube-stats
+          knownHosts: ${{ secrets.STATS_DEPLOYEMENT_KNOWN_HOSTS }}
+          deployKey: ${{ secrets.STATS_DEPLOYEMENT_KEY }}
+          deployUser: ${{ secrets.STATS_DEPLOYEMENT_USER }}
+          deployHost: ${{ secrets.STATS_DEPLOYEMENT_HOST }}
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0366ac49f..030ec3790 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -50,29 +50,11 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Use Node.js
-        uses: actions/setup-node@v1
+      - uses: './.github/actions/reusable-prepare-peertube-build'
         with:
           node-version: '12.x'
 
-      - name: Setup system dependencies
-        run: |
-          sudo apt-get install postgresql-client-common redis-tools parallel
-          wget --quiet --no-check-certificate "https://download.cpy.re/ffmpeg/ffmpeg-release-4.3.1-64bit-static.tar.xz"
-          tar xf ffmpeg-release-4.3.1-64bit-static.tar.xz
-          mkdir -p $HOME/bin
-          cp ffmpeg-*/{ffmpeg,ffprobe} $HOME/bin
-          echo "$HOME/bin" >> $GITHUB_PATH
-
-      - name: Cache Node.js modules
-        uses: actions/cache@v2
-        with:
-          path: |
-            **/node_modules
-          key: ${{ runner.OS }}-node-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.OS }}-node-
-            ${{ runner.OS }}-
+      - uses: './.github/actions/reusable-prepare-peertube-run'
 
       - name: Cache fixtures
         uses: actions/cache@v2
@@ -85,9 +67,6 @@ jobs:
             ${{ runner.OS }}-fixtures-
             ${{ runner.OS }}-
 
-      - name: Install dependencies
-        run: yarn install --frozen-lockfile
-
       - name: Set env test variable (schedule)
         if: github.event_name != 'schedule'
         run: |
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
deleted file mode 100644
index 66c6df761..000000000
--- a/.gitlab-ci.yml
+++ /dev/null
@@ -1,40 +0,0 @@
-image: chocobozzz/peertube-ci:14
-
-stages:
-  - clients
-  - docker-nightly
-
-cache:
-  key: yarn
-  paths:
-    - .yarn-cache
-    - cached-fixtures
-
-# build-openapi-clients:
-#   stage: clients
-#   only:
-#     refs:
-#       - master
-#       - schedules
-#     changes:
-#       - support/doc/api/openapi.yaml
-#   script:
-#     - apt-get update -qq
-#     - apt-get -yqqq install openjdk-8-jre
-#     - yarn install --pure-lockfile
-#     - scripts/openapi-peertube-version.sh
-#     - scripts/openapi-clients.sh
-
-build-nightly:
-  stage: docker-nightly
-  only:
-    - schedules
-  script:
-    - yarn install --pure-lockfile --cache-folder .yarn-cache
-    - npm run nightly
-    - mkdir "${HOME}/.ssh"
-    - chmod 700 "${HOME}/.ssh"
-    - if [ ! -z ${DEPLOYEMENT_KNOWN_HOSTS+x} ]; then echo -e "${DEPLOYEMENT_KNOWN_HOSTS}" > ${HOME}/.ssh/known_hosts; fi
-    - eval `ssh-agent -s`
-    - if [ ! -z ${DEPLOYEMENT_KEY+x} ]; then ssh-add <(echo "${DEPLOYEMENT_KEY}"); fi
-    - if [ ! -z ${DEPLOYEMENT_KEY+x} ]; then scp ./peertube-nightly-* ${DEPLOYEMENT_USER}@${DEPLOYEMENT_HOST}:../../web/nightly; fi
diff --git a/support/doc/development/ci.md b/support/doc/development/ci.md
new file mode 100644
index 000000000..7d6eef197
--- /dev/null
+++ b/support/doc/development/ci.md
@@ -0,0 +1,40 @@
+# Continuous integration
+
+PeerTube uses Github Actions as a CI platform.
+CI tasks are described in `.github/workflows`.
+
+## benchmark.yml
+
+*Scheduled*
+
+Run various benchmarks (build, API etc) and upload results on https://builds.joinpeertube.org/peertube-stats/ to be publicly consumed.
+
+## codeql.yml
+
+*Scheduled, on push on develop and on pull request*
+
+Run CodeQL task to throw code security issues in Github. https://lgtm.com/projects/g/Chocobozzz/PeerTube can also be used.
+
+## docker.yml
+
+*Scheduled and on push on master*
+
+Build `chocobozzz/peertube-webserver:latest`, `chocobozzz/peertube:production-...`, `chocobozzz/peertube:v-...` (only latest PeerTube tag) and `chocobozzz/peertube:develop-...` Docker images. Scheduled to automatically upgrade image software (Debian security issues etc).
+
+## nightly.yml
+
+*Scheduled*
+
+Build PeerTube nightly build (`develop` branch) and upload the release on https://builds.joinpeertube.org/nightly.
+
+## stats.yml
+
+*On push on develop*
+
+Create various PeerTube stats (line of codes, build size, lighthouse report) and upload results on https://builds.joinpeertube.org/peertube-stats/ to be publicly consumed.
+
+## test.yml
+
+*Scheduled, on push and pull request*
+
+Run PeerTube lint and tests.