From bf54587a3e2ad9c2c186828f2a5682b91ee2cc00 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 17 Dec 2021 09:29:23 +0100 Subject: shared/ typescript types dir server-commands --- shared/server-commands/miscs/checks.ts | 58 ++++++++++++ shared/server-commands/miscs/generate.ts | 75 +++++++++++++++ shared/server-commands/miscs/index.ts | 5 + shared/server-commands/miscs/sql-command.ts | 141 ++++++++++++++++++++++++++++ shared/server-commands/miscs/tests.ts | 101 ++++++++++++++++++++ shared/server-commands/miscs/webtorrent.ts | 46 +++++++++ 6 files changed, 426 insertions(+) create mode 100644 shared/server-commands/miscs/checks.ts create mode 100644 shared/server-commands/miscs/generate.ts create mode 100644 shared/server-commands/miscs/index.ts create mode 100644 shared/server-commands/miscs/sql-command.ts create mode 100644 shared/server-commands/miscs/tests.ts create mode 100644 shared/server-commands/miscs/webtorrent.ts (limited to 'shared/server-commands/miscs') diff --git a/shared/server-commands/miscs/checks.ts b/shared/server-commands/miscs/checks.ts new file mode 100644 index 000000000..589928997 --- /dev/null +++ b/shared/server-commands/miscs/checks.ts @@ -0,0 +1,58 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */ + +import { expect } from 'chai' +import { pathExists, readFile } from 'fs-extra' +import { join } from 'path' +import { root } from '@shared/core-utils' +import { HttpStatusCode } from '@shared/models' +import { makeGetRequest } from '../requests' +import { PeerTubeServer } from '../server' + +// Default interval -> 5 minutes +function dateIsValid (dateString: string, interval = 300000) { + const dateToCheck = new Date(dateString) + const now = new Date() + + return Math.abs(now.getTime() - dateToCheck.getTime()) <= interval +} + +function expectStartWith (str: string, start: string) { + expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.true +} + +async function expectLogDoesNotContain (server: PeerTubeServer, str: string) { + const content = await server.servers.getLogContent() + + expect(content.toString()).to.not.contain(str) +} + +async function testImage (url: string, imageName: string, imagePath: string, extension = '.jpg') { + const res = await makeGetRequest({ + url, + path: imagePath, + expectedStatus: HttpStatusCode.OK_200 + }) + + const body = res.body + + const data = await readFile(join(root(), 'server', 'tests', 'fixtures', imageName + extension)) + const minLength = body.length - ((30 * body.length) / 100) + const maxLength = body.length + ((30 * body.length) / 100) + + expect(data.length).to.be.above(minLength, 'the generated image is way smaller than the recorded fixture') + expect(data.length).to.be.below(maxLength, 'the generated image is way larger than the recorded fixture') +} + +async function testFileExistsOrNot (server: PeerTubeServer, directory: string, filePath: string, exist: boolean) { + const base = server.servers.buildDirectory(directory) + + expect(await pathExists(join(base, filePath))).to.equal(exist) +} + +export { + dateIsValid, + testImage, + expectLogDoesNotContain, + testFileExistsOrNot, + expectStartWith +} diff --git a/shared/server-commands/miscs/generate.ts b/shared/server-commands/miscs/generate.ts new file mode 100644 index 000000000..93673a063 --- /dev/null +++ b/shared/server-commands/miscs/generate.ts @@ -0,0 +1,75 @@ +import { expect } from 'chai' +import ffmpeg from 'fluent-ffmpeg' +import { ensureDir, pathExists } from 'fs-extra' +import { dirname } from 'path' +import { getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '@shared/extra-utils/ffprobe' +import { getMaxBitrate } from '@shared/core-utils' +import { buildAbsoluteFixturePath } from './tests' + +async function ensureHasTooBigBitrate (fixturePath: string) { + const bitrate = await getVideoFileBitrate(fixturePath) + const dataResolution = await getVideoFileResolution(fixturePath) + const fps = await getVideoFileFPS(fixturePath) + + const maxBitrate = getMaxBitrate({ ...dataResolution, fps }) + expect(bitrate).to.be.above(maxBitrate) +} + +async function generateHighBitrateVideo () { + const tempFixturePath = buildAbsoluteFixturePath('video_high_bitrate_1080p.mp4', true) + + await ensureDir(dirname(tempFixturePath)) + + const exists = await pathExists(tempFixturePath) + if (!exists) { + console.log('Generating high bitrate video.') + + // Generate a random, high bitrate video on the fly, so we don't have to include + // a large file in the repo. The video needs to have a certain minimum length so + // that FFmpeg properly applies bitrate limits. + // https://stackoverflow.com/a/15795112 + return new Promise((res, rej) => { + ffmpeg() + .outputOptions([ '-f rawvideo', '-video_size 1920x1080', '-i /dev/urandom' ]) + .outputOptions([ '-ac 2', '-f s16le', '-i /dev/urandom', '-t 10' ]) + .outputOptions([ '-maxrate 10M', '-bufsize 10M' ]) + .output(tempFixturePath) + .on('error', rej) + .on('end', () => res(tempFixturePath)) + .run() + }) + } + + await ensureHasTooBigBitrate(tempFixturePath) + + return tempFixturePath +} + +async function generateVideoWithFramerate (fps = 60) { + const tempFixturePath = buildAbsoluteFixturePath(`video_${fps}fps.mp4`, true) + + await ensureDir(dirname(tempFixturePath)) + + const exists = await pathExists(tempFixturePath) + if (!exists) { + console.log('Generating video with framerate %d.', fps) + + return new Promise((res, rej) => { + ffmpeg() + .outputOptions([ '-f rawvideo', '-video_size 1280x720', '-i /dev/urandom' ]) + .outputOptions([ '-ac 2', '-f s16le', '-i /dev/urandom', '-t 10' ]) + .outputOptions([ `-r ${fps}` ]) + .output(tempFixturePath) + .on('error', rej) + .on('end', () => res(tempFixturePath)) + .run() + }) + } + + return tempFixturePath +} + +export { + generateHighBitrateVideo, + generateVideoWithFramerate +} diff --git a/shared/server-commands/miscs/index.ts b/shared/server-commands/miscs/index.ts new file mode 100644 index 000000000..4474661de --- /dev/null +++ b/shared/server-commands/miscs/index.ts @@ -0,0 +1,5 @@ +export * from './checks' +export * from './generate' +export * from './sql-command' +export * from './tests' +export * from './webtorrent' diff --git a/shared/server-commands/miscs/sql-command.ts b/shared/server-commands/miscs/sql-command.ts new file mode 100644 index 000000000..bedb3349b --- /dev/null +++ b/shared/server-commands/miscs/sql-command.ts @@ -0,0 +1,141 @@ +import { QueryTypes, Sequelize } from 'sequelize' +import { AbstractCommand } from '../shared/abstract-command' + +export class SQLCommand extends AbstractCommand { + private sequelize: Sequelize + + deleteAll (table: string) { + const seq = this.getSequelize() + + const options = { type: QueryTypes.DELETE } + + return seq.query(`DELETE FROM "${table}"`, options) + } + + async getCount (table: string) { + const seq = this.getSequelize() + + const options = { type: QueryTypes.SELECT as QueryTypes.SELECT } + + const [ { total } ] = await seq.query<{ total: string }>(`SELECT COUNT(*) as total FROM "${table}"`, options) + if (total === null) return 0 + + return parseInt(total, 10) + } + + setActorField (to: string, field: string, value: string) { + const seq = this.getSequelize() + + const options = { type: QueryTypes.UPDATE } + + return seq.query(`UPDATE actor SET "${field}" = '${value}' WHERE url = '${to}'`, options) + } + + setVideoField (uuid: string, field: string, value: string) { + const seq = this.getSequelize() + + const options = { type: QueryTypes.UPDATE } + + return seq.query(`UPDATE video SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options) + } + + setPlaylistField (uuid: string, field: string, value: string) { + const seq = this.getSequelize() + + const options = { type: QueryTypes.UPDATE } + + return seq.query(`UPDATE "videoPlaylist" SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options) + } + + async countVideoViewsOf (uuid: string) { + const seq = this.getSequelize() + + const query = 'SELECT SUM("videoView"."views") AS "total" FROM "videoView" ' + + `INNER JOIN "video" ON "video"."id" = "videoView"."videoId" WHERE "video"."uuid" = '${uuid}'` + + const options = { type: QueryTypes.SELECT as QueryTypes.SELECT } + const [ { total } ] = await seq.query<{ total: number }>(query, options) + + if (!total) return 0 + + return parseInt(total + '', 10) + } + + getActorImage (filename: string) { + return this.selectQuery(`SELECT * FROM "actorImage" WHERE filename = '${filename}'`) + .then(rows => rows[0]) + } + + selectQuery (query: string) { + const seq = this.getSequelize() + const options = { type: QueryTypes.SELECT as QueryTypes.SELECT } + + return seq.query(query, options) + } + + updateQuery (query: string) { + const seq = this.getSequelize() + const options = { type: QueryTypes.UPDATE as QueryTypes.UPDATE } + + return seq.query(query, options) + } + + setPluginField (pluginName: string, field: string, value: string) { + const seq = this.getSequelize() + + const options = { type: QueryTypes.UPDATE } + + return seq.query(`UPDATE "plugin" SET "${field}" = '${value}' WHERE "name" = '${pluginName}'`, options) + } + + setPluginVersion (pluginName: string, newVersion: string) { + return this.setPluginField(pluginName, 'version', newVersion) + } + + setPluginLatestVersion (pluginName: string, newVersion: string) { + return this.setPluginField(pluginName, 'latestVersion', newVersion) + } + + setActorFollowScores (newScore: number) { + const seq = this.getSequelize() + + const options = { type: QueryTypes.UPDATE } + + return seq.query(`UPDATE "actorFollow" SET "score" = ${newScore}`, options) + } + + setTokenField (accessToken: string, field: string, value: string) { + const seq = this.getSequelize() + + const options = { type: QueryTypes.UPDATE } + + return seq.query(`UPDATE "oAuthToken" SET "${field}" = '${value}' WHERE "accessToken" = '${accessToken}'`, options) + } + + async cleanup () { + if (!this.sequelize) return + + await this.sequelize.close() + this.sequelize = undefined + } + + private getSequelize () { + if (this.sequelize) return this.sequelize + + const dbname = 'peertube_test' + this.server.internalServerNumber + const username = 'peertube' + const password = 'peertube' + const host = 'localhost' + const port = 5432 + + this.sequelize = new Sequelize(dbname, username, password, { + dialect: 'postgres', + host, + port, + logging: false + }) + + return this.sequelize + } + +} diff --git a/shared/server-commands/miscs/tests.ts b/shared/server-commands/miscs/tests.ts new file mode 100644 index 000000000..658fe5fd3 --- /dev/null +++ b/shared/server-commands/miscs/tests.ts @@ -0,0 +1,101 @@ +import { stat } from 'fs-extra' +import { basename, isAbsolute, join, resolve } from 'path' + +const FIXTURE_URLS = { + peertube_long: 'https://peertube2.cpy.re/videos/watch/122d093a-1ede-43bd-bd34-59d2931ffc5e', + peertube_short: 'https://peertube2.cpy.re/w/3fbif9S3WmtTP8gGsC5HBd', + + youtube: 'https://www.youtube.com/watch?v=msX3jv1XdvM', + + /** + * The video is used to check format-selection correctness wrt. HDR, + * which brings its own set of oddities outside of a MediaSource. + * + * The video needs to have the following format_ids: + * (which you can check by using `youtube-dl -F`): + * - (webm vp9) + * - (mp4 avc1) + * - (webm vp9.2 HDR) + */ + youtubeHDR: 'https://www.youtube.com/watch?v=RQgnBB9z_N4', + + // eslint-disable-next-line max-len + magnet: 'magnet:?xs=https%3A%2F%2Fpeertube2.cpy.re%2Flazy-static%2Ftorrents%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.torrent&xt=urn:btih:0f498834733e8057ed5c6f2ee2b4efd8d84a76ee&dn=super+peertube2+video&tr=https%3A%2F%2Fpeertube2.cpy.re%2Ftracker%2Fannounce&tr=wss%3A%2F%2Fpeertube2.cpy.re%3A443%2Ftracker%2Fsocket&ws=https%3A%2F%2Fpeertube2.cpy.re%2Fstatic%2Fwebseed%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.mp4', + + badVideo: 'https://download.cpy.re/peertube/bad_video.mp4', + goodVideo: 'https://download.cpy.re/peertube/good_video.mp4', + goodVideo720: 'https://download.cpy.re/peertube/good_video_720.mp4', + + file4K: 'https://download.cpy.re/peertube/4k_file.txt' +} + +function parallelTests () { + return process.env.MOCHA_PARALLEL === 'true' +} + +function isGithubCI () { + return !!process.env.GITHUB_WORKSPACE +} + +function areHttpImportTestsDisabled () { + const disabled = process.env.DISABLE_HTTP_IMPORT_TESTS === 'true' + + if (disabled) console.log('DISABLE_HTTP_IMPORT_TESTS env set to "true" so import tests are disabled') + + return disabled +} + +function areObjectStorageTestsDisabled () { + const disabled = process.env.ENABLE_OBJECT_STORAGE_TESTS !== 'true' + + if (disabled) console.log('ENABLE_OBJECT_STORAGE_TESTS env is not set to "true" so object storage tests are disabled') + + return disabled +} + +function buildAbsoluteFixturePath (path: string, customCIPath = false) { + if (isAbsolute(path)) return path + + if (customCIPath && process.env.GITHUB_WORKSPACE) { + return join(process.env.GITHUB_WORKSPACE, 'fixtures', path) + } + + return join(root(), 'server', 'tests', 'fixtures', path) +} + +function root () { + // We are in /miscs + let root = join(__dirname, '..', '..', '..') + + if (basename(root) === 'dist') root = resolve(root, '..') + + return root +} + +function wait (milliseconds: number) { + return new Promise(resolve => setTimeout(resolve, milliseconds)) +} + +async function getFileSize (path: string) { + const stats = await stat(path) + + return stats.size +} + +function buildRequestStub (): any { + return { } +} + +export { + FIXTURE_URLS, + + parallelTests, + isGithubCI, + areHttpImportTestsDisabled, + buildAbsoluteFixturePath, + getFileSize, + buildRequestStub, + areObjectStorageTestsDisabled, + wait, + root +} diff --git a/shared/server-commands/miscs/webtorrent.ts b/shared/server-commands/miscs/webtorrent.ts new file mode 100644 index 000000000..0683f8893 --- /dev/null +++ b/shared/server-commands/miscs/webtorrent.ts @@ -0,0 +1,46 @@ +import { readFile } from 'fs-extra' +import parseTorrent from 'parse-torrent' +import { basename, join } from 'path' +import * as WebTorrent from 'webtorrent' +import { VideoFile } from '@shared/models' +import { PeerTubeServer } from '../server' + +let webtorrent: WebTorrent.Instance + +function webtorrentAdd (torrentId: string, refreshWebTorrent = false) { + const WebTorrent = require('webtorrent') + + if (webtorrent && refreshWebTorrent) webtorrent.destroy() + if (!webtorrent || refreshWebTorrent) webtorrent = new WebTorrent() + + webtorrent.on('error', err => console.error('Error in webtorrent', err)) + + return new Promise(res => { + const torrent = webtorrent.add(torrentId, res) + + torrent.on('error', err => console.error('Error in webtorrent torrent', err)) + torrent.on('warning', warn => { + const msg = typeof warn === 'string' + ? warn + : warn.message + + if (msg.includes('Unsupported')) return + + console.error('Warning in webtorrent torrent', warn) + }) + }) +} + +async function parseTorrentVideo (server: PeerTubeServer, file: VideoFile) { + const torrentName = basename(file.torrentUrl) + const torrentPath = server.servers.buildDirectory(join('torrents', torrentName)) + + const data = await readFile(torrentPath) + + return parseTorrent(data) +} + +export { + webtorrentAdd, + parseTorrentVideo +} -- cgit v1.2.3