6 files changed, 426 insertions, 0 deletions

diff --git a/shared/server-commands/miscs/checks.ts b/shared/server-commands/miscs/checks.ts
new file mode 100644
index 000000000..589928997
--- /dev/null
+++ b/shared/server-commands/miscs/checks.ts
@@ -0,0 +1,58 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */
+
+import { expect } from 'chai'
+import { pathExists, readFile } from 'fs-extra'
+import { join } from 'path'
+import { root } from '@shared/core-utils'
+import { HttpStatusCode } from '@shared/models'
+import { makeGetRequest } from '../requests'
+import { PeerTubeServer } from '../server'
+
+// Default interval -> 5 minutes
+function dateIsValid (dateString: string, interval = 300000) {
+  const dateToCheck = new Date(dateString)
+  const now = new Date()
+
+  return Math.abs(now.getTime() - dateToCheck.getTime()) <= interval
+}
+
+function expectStartWith (str: string, start: string) {
+  expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.true
+}
+
+async function expectLogDoesNotContain (server: PeerTubeServer, str: string) {
+  const content = await server.servers.getLogContent()
+
+  expect(content.toString()).to.not.contain(str)
+}
+
+async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') {
+  const res = await makeGetRequest({
+    url,
+    path: imagePath,
+    expectedStatus: HttpStatusCode.OK_200
+  })
+
+  const body = res.body
+
+  const data = await readFile(join(root(), 'server', 'tests', 'fixtures', imageName + extension))
+  const minLength = body.length - ((30 * body.length) / 100)
+  const maxLength = body.length + ((30 * body.length) / 100)
+
+  expect(data.length).to.be.above(minLength, 'the generated image is way smaller than the recorded fixture')
+  expect(data.length).to.be.below(maxLength, 'the generated image is way larger than the recorded fixture')
+}
+
+async function testFileExistsOrNot (server: PeerTubeServer, directory: string, filePath: string, exist: boolean) {
+  const base = server.servers.buildDirectory(directory)
+
+  expect(await pathExists(join(base, filePath))).to.equal(exist)
+}
+
+export {
+  dateIsValid,
+  testImage,
+  expectLogDoesNotContain,
+  testFileExistsOrNot,
+  expectStartWith
+}
diff --git a/shared/server-commands/miscs/generate.ts b/shared/server-commands/miscs/generate.ts
new file mode 100644
index 000000000..93673a063
--- /dev/null
+++ b/shared/server-commands/miscs/generate.ts
@@ -0,0 +1,75 @@
+import { expect } from 'chai'
+import ffmpeg from 'fluent-ffmpeg'
+import { ensureDir, pathExists } from 'fs-extra'
+import { dirname } from 'path'
+import { getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '@shared/extra-utils/ffprobe'
+import { getMaxBitrate } from '@shared/core-utils'
+import { buildAbsoluteFixturePath } from './tests'
+
+async function ensureHasTooBigBitrate (fixturePath: string) {
+  const bitrate = await getVideoFileBitrate(fixturePath)
+  const dataResolution = await getVideoFileResolution(fixturePath)
+  const fps = await getVideoFileFPS(fixturePath)
+
+  const maxBitrate = getMaxBitrate({ ...dataResolution, fps })
+  expect(bitrate).to.be.above(maxBitrate)
+}
+
+async function generateHighBitrateVideo () {
+  const tempFixturePath = buildAbsoluteFixturePath('video_high_bitrate_1080p.mp4', true)
+
+  await ensureDir(dirname(tempFixturePath))
+
+  const exists = await pathExists(tempFixturePath)
+  if (!exists) {
+    console.log('Generating high bitrate video.')
+
+    // Generate a random, high bitrate video on the fly, so we don't have to include
+    // a large file in the repo. The video needs to have a certain minimum length so
+    // that FFmpeg properly applies bitrate limits.
+    // https://stackoverflow.com/a/15795112
+    return new Promise<string>((res, rej) => {
+      ffmpeg()
+        .outputOptions([ '-f rawvideo', '-video_size 1920x1080', '-i /dev/urandom' ])
+        .outputOptions([ '-ac 2', '-f s16le', '-i /dev/urandom', '-t 10' ])
+        .outputOptions([ '-maxrate 10M', '-bufsize 10M' ])
+        .output(tempFixturePath)
+        .on('error', rej)
+        .on('end', () => res(tempFixturePath))
+        .run()
+    })
+  }
+
+  await ensureHasTooBigBitrate(tempFixturePath)
+
+  return tempFixturePath
+}
+
+async function generateVideoWithFramerate (fps = 60) {
+  const tempFixturePath = buildAbsoluteFixturePath(`video_${fps}fps.mp4`, true)
+
+  await ensureDir(dirname(tempFixturePath))
+
+  const exists = await pathExists(tempFixturePath)
+  if (!exists) {
+    console.log('Generating video with framerate %d.', fps)
+
+    return new Promise<string>((res, rej) => {
+      ffmpeg()
+        .outputOptions([ '-f rawvideo', '-video_size 1280x720', '-i /dev/urandom' ])
+        .outputOptions([ '-ac 2', '-f s16le', '-i /dev/urandom', '-t 10' ])
+        .outputOptions([ `-r ${fps}` ])
+        .output(tempFixturePath)
+        .on('error', rej)
+        .on('end', () => res(tempFixturePath))
+        .run()
+    })
+  }
+
+  return tempFixturePath
+}
+
+export {
+  generateHighBitrateVideo,
+  generateVideoWithFramerate
+}
diff --git a/shared/server-commands/miscs/index.ts b/shared/server-commands/miscs/index.ts
new file mode 100644
index 000000000..4474661de
--- /dev/null
+++ b/shared/server-commands/miscs/index.ts
@@ -0,0 +1,5 @@
+export * from './checks'
+export * from './generate'
+export * from './sql-command'
+export * from './tests'
+export * from './webtorrent'
diff --git a/shared/server-commands/miscs/sql-command.ts b/shared/server-commands/miscs/sql-command.ts
new file mode 100644
index 000000000..bedb3349b
--- /dev/null
+++ b/shared/server-commands/miscs/sql-command.ts
@@ -0,0 +1,141 @@
+import { QueryTypes, Sequelize } from 'sequelize'
+import { AbstractCommand } from '../shared/abstract-command'
+
+export class SQLCommand extends AbstractCommand {
+  private sequelize: Sequelize
+
+  deleteAll (table: string) {
+    const seq = this.getSequelize()
+
+    const options = { type: QueryTypes.DELETE }
+
+    return seq.query(`DELETE FROM "${table}"`, options)
+  }
+
+  async getCount (table: string) {
+    const seq = this.getSequelize()
+
+    const options = { type: QueryTypes.SELECT as QueryTypes.SELECT }
+
+    const [ { total } ] = await seq.query<{ total: string }>(`SELECT COUNT(*) as total FROM "${table}"`, options)
+    if (total === null) return 0
+
+    return parseInt(total, 10)
+  }
+
+  setActorField (to: string, field: string, value: string) {
+    const seq = this.getSequelize()
+
+    const options = { type: QueryTypes.UPDATE }
+
+    return seq.query(`UPDATE actor SET "${field}" = '${value}' WHERE url = '${to}'`, options)
+  }
+
+  setVideoField (uuid: string, field: string, value: string) {
+    const seq = this.getSequelize()
+
+    const options = { type: QueryTypes.UPDATE }
+
+    return seq.query(`UPDATE video SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options)
+  }
+
+  setPlaylistField (uuid: string, field: string, value: string) {
+    const seq = this.getSequelize()
+
+    const options = { type: QueryTypes.UPDATE }
+
+    return seq.query(`UPDATE "videoPlaylist" SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options)
+  }
+
+  async countVideoViewsOf (uuid: string) {
+    const seq = this.getSequelize()
+
+    const query = 'SELECT SUM("videoView"."views") AS "total" FROM "videoView" ' +
+      `INNER JOIN "video" ON "video"."id" = "videoView"."videoId" WHERE "video"."uuid" = '${uuid}'`
+
+    const options = { type: QueryTypes.SELECT as QueryTypes.SELECT }
+    const [ { total } ] = await seq.query<{ total: number }>(query, options)
+
+    if (!total) return 0
+
+    return parseInt(total + '', 10)
+  }
+
+  getActorImage (filename: string) {
+    return this.selectQuery(`SELECT * FROM "actorImage" WHERE filename = '${filename}'`)
+      .then(rows => rows[0])
+  }
+
+  selectQuery (query: string) {
+    const seq = this.getSequelize()
+    const options = { type: QueryTypes.SELECT as QueryTypes.SELECT }
+
+    return seq.query<any>(query, options)
+  }
+
+  updateQuery (query: string) {
+    const seq = this.getSequelize()
+    const options = { type: QueryTypes.UPDATE as QueryTypes.UPDATE }
+
+    return seq.query(query, options)
+  }
+
+  setPluginField (pluginName: string, field: string, value: string) {
+    const seq = this.getSequelize()
+
+    const options = { type: QueryTypes.UPDATE }
+
+    return seq.query(`UPDATE "plugin" SET "${field}" = '${value}' WHERE "name" = '${pluginName}'`, options)
+  }
+
+  setPluginVersion (pluginName: string, newVersion: string) {
+    return this.setPluginField(pluginName, 'version', newVersion)
+  }
+
+  setPluginLatestVersion (pluginName: string, newVersion: string) {
+    return this.setPluginField(pluginName, 'latestVersion', newVersion)
+  }
+
+  setActorFollowScores (newScore: number) {
+    const seq = this.getSequelize()
+
+    const options = { type: QueryTypes.UPDATE }
+
+    return seq.query(`UPDATE "actorFollow" SET "score" = ${newScore}`, options)
+  }
+
+  setTokenField (accessToken: string, field: string, value: string) {
+    const seq = this.getSequelize()
+
+    const options = { type: QueryTypes.UPDATE }
+
+    return seq.query(`UPDATE "oAuthToken" SET "${field}" = '${value}' WHERE "accessToken" = '${accessToken}'`, options)
+  }
+
+  async cleanup () {
+    if (!this.sequelize) return
+
+    await this.sequelize.close()
+    this.sequelize = undefined
+  }
+
+  private getSequelize () {
+    if (this.sequelize) return this.sequelize
+
+    const dbname = 'peertube_test' + this.server.internalServerNumber
+    const username = 'peertube'
+    const password = 'peertube'
+    const host = 'localhost'
+    const port = 5432
+
+    this.sequelize = new Sequelize(dbname, username, password, {
+      dialect: 'postgres',
+      host,
+      port,
+      logging: false
+    })
+
+    return this.sequelize
+  }
+
+}
diff --git a/shared/server-commands/miscs/tests.ts b/shared/server-commands/miscs/tests.ts
new file mode 100644
index 000000000..658fe5fd3
--- /dev/null
+++ b/shared/server-commands/miscs/tests.ts
@@ -0,0 +1,101 @@
+import { stat } from 'fs-extra'
+import { basename, isAbsolute, join, resolve } from 'path'
+
+const FIXTURE_URLS = {
+  peertube_long: 'https://peertube2.cpy.re/videos/watch/122d093a-1ede-43bd-bd34-59d2931ffc5e',
+  peertube_short: 'https://peertube2.cpy.re/w/3fbif9S3WmtTP8gGsC5HBd',
+
+  youtube: 'https://www.youtube.com/watch?v=msX3jv1XdvM',
+
+  /**
+   * The video is used to check format-selection correctness wrt. HDR,
+   * which brings its own set of oddities outside of a MediaSource.
+   *
+   * The video needs to have the following format_ids:
+   * (which you can check by using `youtube-dl <url> -F`):
+   * - (webm vp9)
+   * - (mp4 avc1)
+   * - (webm vp9.2 HDR)
+   */
+  youtubeHDR: 'https://www.youtube.com/watch?v=RQgnBB9z_N4',
+
+  // eslint-disable-next-line max-len
+  magnet: 'magnet:?xs=https%3A%2F%2Fpeertube2.cpy.re%2Flazy-static%2Ftorrents%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.torrent&xt=urn:btih:0f498834733e8057ed5c6f2ee2b4efd8d84a76ee&dn=super+peertube2+video&tr=https%3A%2F%2Fpeertube2.cpy.re%2Ftracker%2Fannounce&tr=wss%3A%2F%2Fpeertube2.cpy.re%3A443%2Ftracker%2Fsocket&ws=https%3A%2F%2Fpeertube2.cpy.re%2Fstatic%2Fwebseed%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.mp4',
+
+  badVideo: 'https://download.cpy.re/peertube/bad_video.mp4',
+  goodVideo: 'https://download.cpy.re/peertube/good_video.mp4',
+  goodVideo720: 'https://download.cpy.re/peertube/good_video_720.mp4',
+
+  file4K: 'https://download.cpy.re/peertube/4k_file.txt'
+}
+
+function parallelTests () {
+  return process.env.MOCHA_PARALLEL === 'true'
+}
+
+function isGithubCI () {
+  return !!process.env.GITHUB_WORKSPACE
+}
+
+function areHttpImportTestsDisabled () {
+  const disabled = process.env.DISABLE_HTTP_IMPORT_TESTS === 'true'
+
+  if (disabled) console.log('DISABLE_HTTP_IMPORT_TESTS env set to "true" so import tests are disabled')
+
+  return disabled
+}
+
+function areObjectStorageTestsDisabled () {
+  const disabled = process.env.ENABLE_OBJECT_STORAGE_TESTS !== 'true'
+
+  if (disabled) console.log('ENABLE_OBJECT_STORAGE_TESTS env is not set to "true" so object storage tests are disabled')
+
+  return disabled
+}
+
+function buildAbsoluteFixturePath (path: string, customCIPath = false) {
+  if (isAbsolute(path)) return path
+
+  if (customCIPath && process.env.GITHUB_WORKSPACE) {
+    return join(process.env.GITHUB_WORKSPACE, 'fixtures', path)
+  }
+
+  return join(root(), 'server', 'tests', 'fixtures', path)
+}
+
+function root () {
+  // We are in /miscs
+  let root = join(__dirname, '..', '..', '..')
+
+  if (basename(root) === 'dist') root = resolve(root, '..')
+
+  return root
+}
+
+function wait (milliseconds: number) {
+  return new Promise(resolve => setTimeout(resolve, milliseconds))
+}
+
+async function getFileSize (path: string) {
+  const stats = await stat(path)
+
+  return stats.size
+}
+
+function buildRequestStub (): any {
+  return { }
+}
+
+export {
+  FIXTURE_URLS,
+
+  parallelTests,
+  isGithubCI,
+  areHttpImportTestsDisabled,
+  buildAbsoluteFixturePath,
+  getFileSize,
+  buildRequestStub,
+  areObjectStorageTestsDisabled,
+  wait,
+  root
+}
diff --git a/shared/server-commands/miscs/webtorrent.ts b/shared/server-commands/miscs/webtorrent.ts
new file mode 100644
index 000000000..0683f8893
--- /dev/null
+++ b/shared/server-commands/miscs/webtorrent.ts
@@ -0,0 +1,46 @@
+import { readFile } from 'fs-extra'
+import parseTorrent from 'parse-torrent'
+import { basename, join } from 'path'
+import * as WebTorrent from 'webtorrent'
+import { VideoFile } from '@shared/models'
+import { PeerTubeServer } from '../server'
+
+let webtorrent: WebTorrent.Instance
+
+function webtorrentAdd (torrentId: string, refreshWebTorrent = false) {
+  const WebTorrent = require('webtorrent')
+
+  if (webtorrent && refreshWebTorrent) webtorrent.destroy()
+  if (!webtorrent || refreshWebTorrent) webtorrent = new WebTorrent()
+
+  webtorrent.on('error', err => console.error('Error in webtorrent', err))
+
+  return new Promise<WebTorrent.Torrent>(res => {
+    const torrent = webtorrent.add(torrentId, res)
+
+    torrent.on('error', err => console.error('Error in webtorrent torrent', err))
+    torrent.on('warning', warn => {
+      const msg = typeof warn === 'string'
+        ? warn
+        : warn.message
+
+      if (msg.includes('Unsupported')) return
+
+      console.error('Warning in webtorrent torrent', warn)
+    })
+  })
+}
+
+async function parseTorrentVideo (server: PeerTubeServer, file: VideoFile) {
+  const torrentName = basename(file.torrentUrl)
+  const torrentPath = server.servers.buildDirectory(join('torrents', torrentName))
+
+  const data = await readFile(torrentPath)
+
+  return parseTorrent(data)
+}
+
+export {
+  webtorrentAdd,
+  parseTorrentVideo
+}