From 3e17515e2996b79e23f569c296051a91af3fcbe4 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 7 Aug 2018 15:17:17 +0200 Subject: Add torrent tests --- server/controllers/api/videos/import.ts | 6 +- server/helpers/core-utils.ts | 5 +- server/helpers/utils.ts | 3 +- server/helpers/webtorrent.ts | 9 +- server/lib/job-queue/handlers/video-import.ts | 8 +- server/tests/api/check-params/users.ts | 2 +- server/tests/api/check-params/video-imports.ts | 2 +- server/tests/api/videos/video-imports.ts | 159 +++++++++++++++++++------ server/tests/fixtures/60fps_small-240p.torrent | 1 - server/tests/fixtures/video-720p.torrent | Bin 0 -> 2644 bytes server/tests/utils/videos/video-imports.ts | 10 +- 11 files changed, 145 insertions(+), 60 deletions(-) delete mode 100644 server/tests/fixtures/60fps_small-240p.torrent create mode 100644 server/tests/fixtures/video-720p.torrent diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index 94dafcdbd..b2f73fa48 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts @@ -97,7 +97,7 @@ async function addTorrentImport (req: express.Request, res: express.Response, to await processThumbnail(req, video) await processPreview(req, video) - const tags = null + const tags = body.tags || undefined const videoImportAttributes = { magnetUri, torrentName, @@ -224,11 +224,13 @@ function insertIntoDB ( videoCreated.VideoChannel = videoChannel // Set tags to the video - if (tags !== undefined) { + if (tags) { const tagInstances = await TagModel.findOrCreateTags(tags, t) await videoCreated.$set('Tags', tagInstances, sequelizeOptions) videoCreated.Tags = tagInstances + } else { + videoCreated.Tags = [] } // Create video import object in database diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 25eb6454a..3b38da66c 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts @@ -5,7 +5,7 @@ import * as bcrypt from 'bcrypt' import * as createTorrent from 'create-torrent' -import { pseudoRandomBytes } from 'crypto' +import { createHash, pseudoRandomBytes } from 'crypto' import { copyFile, readdir, readFile, rename, stat, Stats, unlink, writeFile } from 'fs' import * as mkdirp from 'mkdirp' import { isAbsolute, join } from 'path' @@ -13,7 +13,6 @@ import * as pem from 'pem' import * as rimraf from 'rimraf' import { URL } from 'url' import { truncate } from 'lodash' -import * as crypto from 'crypto' function sanitizeUrl (url: string) { const urlObject = new URL(url) @@ -97,7 +96,7 @@ function peertubeTruncate (str: string, maxLength: number) { } function sha256 (str: string) { - return crypto.createHash('sha256').update(str).digest('hex') + return createHash('sha256').update(str).digest('hex') } function promisify0 (func: (cb: (err: any, result: A) => void) => void): () => Promise { diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index 2ad87951e..eaad55555 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts @@ -9,8 +9,7 @@ import { ApplicationModel } from '../models/application/application' import { pseudoRandomBytesPromise, sha256, unlinkPromise } from './core-utils' import { logger } from './logger' import { isArray } from './custom-validators/misc' -import * as crypto from "crypto" -import { join } from "path" +import { join } from 'path' import { Instance as ParseTorrent } from 'parse-torrent' const isCidr = require('is-cidr') diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index 04b3ac71b..121cd0b41 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -2,7 +2,6 @@ import { logger } from './logger' import { generateVideoTmpPath } from './utils' import * as WebTorrent from 'webtorrent' import { createWriteStream } from 'fs' -import { Instance as ParseTorrent } from 'parse-torrent' import { CONFIG } from '../initializers' import { join } from 'path' @@ -20,10 +19,12 @@ function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: stri if (torrent.files.length !== 1) return rej(new Error('The number of files is not equal to 1 for ' + torrentId)) const file = torrent.files[ 0 ] - file.createReadStream().pipe(createWriteStream(path)) - }) - torrent.on('done', () => res(path)) + const writeStream = createWriteStream(path) + writeStream.on('finish', () => res(path)) + + file.createReadStream().pipe(writeStream) + }) torrent.on('error', err => rej(err)) }) diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 28a03d19e..2d19b82a4 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -114,8 +114,8 @@ async function processFile (downloader: () => Promise, videoImport: Vide tempVideoPath = await downloader() // Get information about this video - const { size } = await statPromise(tempVideoPath) - const isAble = await videoImport.User.isAbleToUploadVideo({ size }) + const stats = await statPromise(tempVideoPath) + const isAble = await videoImport.User.isAbleToUploadVideo({ size: stats.size }) if (isAble === false) { throw new Error('The user video quota is exceeded with this video to import.') } @@ -128,7 +128,7 @@ async function processFile (downloader: () => Promise, videoImport: Vide const videoFileData = { extname: extname(tempVideoPath), resolution: videoFileResolution, - size, + size: stats.size, fps, videoId: videoImport.videoId } @@ -209,7 +209,7 @@ async function processFile (downloader: () => Promise, videoImport: Vide } catch (err) { try { - if (tempVideoPath) await unlinkPromise(tempVideoPath) + // if (tempVideoPath) await unlinkPromise(tempVideoPath) } catch (errUnlink) { logger.warn('Cannot cleanup files after a video import error.', { err: errUnlink }) } diff --git a/server/tests/api/check-params/users.ts b/server/tests/api/check-params/users.ts index e1655e85b..7b25df29f 100644 --- a/server/tests/api/check-params/users.ts +++ b/server/tests/api/check-params/users.ts @@ -627,7 +627,7 @@ describe('Test users API validators', function () { } await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { targetUrl: getYoutubeVideoUrl() })) await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { magnetUri: getMagnetURI() })) - await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { torrentfile: '60fps_small-240p.torrent' })) + await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { torrentfile: 'video-720p.torrent' })) await waitJobs([ server ]) diff --git a/server/tests/api/check-params/video-imports.ts b/server/tests/api/check-params/video-imports.ts index e62f0918e..38ddd4e56 100644 --- a/server/tests/api/check-params/video-imports.ts +++ b/server/tests/api/check-params/video-imports.ts @@ -303,7 +303,7 @@ describe('Test video imports API validator', function () { fields = omit(fields, 'magnetUri') const attaches = { - 'torrentfile': join(__dirname, '..', '..', 'fixtures', '60fps_small-240p.torrent') + 'torrentfile': join(__dirname, '..', '..', 'fixtures', 'video-720p.torrent') } await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches, statusCodeExpected: 409 }) diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts index f21ade5c3..b7866d529 100644 --- a/server/tests/api/videos/video-imports.ts +++ b/server/tests/api/videos/video-imports.ts @@ -2,7 +2,7 @@ import * as chai from 'chai' import 'mocha' -import { VideoDetails, VideoPrivacy } from '../../../../shared/models/videos' +import { VideoDetails, VideoImport, VideoPrivacy } from '../../../../shared/models/videos' import { doubleFollow, flushAndRunMultipleServers, @@ -10,12 +10,13 @@ import { getMyVideos, getVideo, getVideosList, + immutableAssign, killallServers, ServerInfo, setAccessTokensToServers } from '../../utils' import { waitJobs } from '../../utils/server/jobs' -import { getMyVideoImports, getYoutubeVideoUrl, importVideo } from '../../utils/videos/video-imports' +import { getMagnetURI, getYoutubeVideoUrl, importVideo, getMyVideoImports } from '../../utils/videos/video-imports' const expect = chai.expect @@ -24,19 +25,36 @@ describe('Test video imports', function () { let channelIdServer1: number let channelIdServer2: number - async function checkVideoServer1 (url: string, id: number | string) { - const res = await getVideo(url, id) - const video: VideoDetails = res.body - - expect(video.name).to.equal('small video - youtube') - expect(video.category.label).to.equal('News') - expect(video.licence.label).to.equal('Attribution') - expect(video.language.label).to.equal('Unknown') - expect(video.nsfw).to.be.false - expect(video.description).to.equal('this is a super description') - expect(video.tags).to.deep.equal([ 'tag1', 'tag2' ]) + async function checkVideosServer1 (url: string, idHttp: string, idMagnet: string, idTorrent: string) { + const resHttp = await getVideo(url, idHttp) + const videoHttp: VideoDetails = resHttp.body + + expect(videoHttp.name).to.equal('small video - youtube') + expect(videoHttp.category.label).to.equal('News') + expect(videoHttp.licence.label).to.equal('Attribution') + expect(videoHttp.language.label).to.equal('Unknown') + expect(videoHttp.nsfw).to.be.false + expect(videoHttp.description).to.equal('this is a super description') + expect(videoHttp.tags).to.deep.equal([ 'tag1', 'tag2' ]) + expect(videoHttp.files).to.have.lengthOf(1) + + const resMagnet = await getVideo(url, idMagnet) + const videoMagnet: VideoDetails = resMagnet.body + const resTorrent = await getVideo(url, idTorrent) + const videoTorrent: VideoDetails = resTorrent.body + + for (const video of [ videoMagnet, videoTorrent ]) { + expect(video.category.label).to.equal('Misc') + expect(video.licence.label).to.equal('Unknown') + expect(video.language.label).to.equal('Unknown') + expect(video.nsfw).to.be.false + expect(video.description).to.equal('this is a super torrent description') + expect(video.tags).to.deep.equal([ 'tag_torrent1', 'tag_torrent2' ]) + expect(video.files).to.have.lengthOf(1) + } - expect(video.files).to.have.lengthOf(1) + expect(videoTorrent.name).to.contain('你好 世界 720p.mp4') + expect(videoMagnet.name).to.contain('super peertube2 video') } async function checkVideoServer2 (url: string, id: number | string) { @@ -75,50 +93,88 @@ describe('Test video imports', function () { await doubleFollow(servers[0], servers[1]) }) - it('Should import a video on server 1', async function () { + it('Should import videos on server 1', async function () { this.timeout(60000) - const attributes = { - targetUrl: getYoutubeVideoUrl(), + const baseAttributes = { channelId: channelIdServer1, privacy: VideoPrivacy.PUBLIC } - const res = await importVideo(servers[0].url, servers[0].accessToken, attributes) - expect(res.body.video.name).to.equal('small video - youtube') + + { + const attributes = immutableAssign(baseAttributes, { targetUrl: getYoutubeVideoUrl() }) + const res = await importVideo(servers[0].url, servers[0].accessToken, attributes) + expect(res.body.video.name).to.equal('small video - youtube') + } + + { + const attributes = immutableAssign(baseAttributes, { + magnetUri: getMagnetURI(), + description: 'this is a super torrent description', + tags: [ 'tag_torrent1', 'tag_torrent2' ] + }) + const res = await importVideo(servers[0].url, servers[0].accessToken, attributes) + expect(res.body.video.name).to.equal('super peertube2 video') + } + + { + const attributes = immutableAssign(baseAttributes, { + torrentfile: 'video-720p.torrent', + description: 'this is a super torrent description', + tags: [ 'tag_torrent1', 'tag_torrent2' ] + }) + const res = await importVideo(servers[0].url, servers[0].accessToken, attributes) + expect(res.body.video.name).to.equal('你好 世界 720p.mp4') + } }) - it('Should list the video to import in my videos on server 1', async function () { - const res = await getMyVideos(servers[0].url, servers[0].accessToken, 0, 5) + it('Should list the videos to import in my videos on server 1', async function () { + const res = await getMyVideos(servers[0].url, servers[0].accessToken, 0, 5, 'createdAt') - expect(res.body.total).to.equal(1) + expect(res.body.total).to.equal(3) const videos = res.body.data - expect(videos).to.have.lengthOf(1) + expect(videos).to.have.lengthOf(3) expect(videos[0].name).to.equal('small video - youtube') + expect(videos[1].name).to.equal('super peertube2 video') + expect(videos[2].name).to.equal('你好 世界 720p.mp4') }) - it('Should list the video to import in my imports on server 1', async function () { - const res = await getMyVideoImports(servers[0].url, servers[0].accessToken) + it('Should list the videos to import in my imports on server 1', async function () { + const res = await getMyVideoImports(servers[0].url, servers[0].accessToken, '-createdAt') + + expect(res.body.total).to.equal(3) + const videoImports: VideoImport[] = res.body.data + expect(videoImports).to.have.lengthOf(3) + + expect(videoImports[2].targetUrl).to.equal(getYoutubeVideoUrl()) + expect(videoImports[2].magnetUri).to.be.null + expect(videoImports[2].torrentName).to.be.null + expect(videoImports[2].video.name).to.equal('small video - youtube') - expect(res.body.total).to.equal(1) - const videoImports = res.body.data - expect(videoImports).to.have.lengthOf(1) + expect(videoImports[1].targetUrl).to.be.null + expect(videoImports[1].magnetUri).to.equal(getMagnetURI()) + expect(videoImports[1].torrentName).to.be.null + expect(videoImports[1].video.name).to.equal('super peertube2 video') - expect(videoImports[0].targetUrl).to.equal(getYoutubeVideoUrl()) - expect(videoImports[0].video.name).to.equal('small video - youtube') + expect(videoImports[0].targetUrl).to.be.null + expect(videoImports[0].magnetUri).to.be.null + expect(videoImports[0].torrentName).to.equal('video-720p.torrent') + expect(videoImports[0].video.name).to.equal('你好 世界 720p.mp4') }) - it('Should have the video listed on the two instances1', async function () { + it('Should have the video listed on the two instances', async function () { this.timeout(120000) await waitJobs(servers) for (const server of servers) { const res = await getVideosList(server.url) - expect(res.body.total).to.equal(1) - expect(res.body.data).to.have.lengthOf(1) + expect(res.body.total).to.equal(3) + expect(res.body.data).to.have.lengthOf(3) - await checkVideoServer1(server.url, res.body.data[0].uuid) + const [ videoHttp, videoMagnet, videoTorrent ] = res.body.data + await checkVideosServer1(server.url, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid) } }) @@ -127,7 +183,7 @@ describe('Test video imports', function () { const attributes = { targetUrl: getYoutubeVideoUrl(), - channelId: channelIdServer1, + channelId: channelIdServer2, privacy: VideoPrivacy.PUBLIC, category: 10, licence: 7, @@ -140,18 +196,43 @@ describe('Test video imports', function () { expect(res.body.video.name).to.equal('my super name') }) - it('Should have the video listed on the two instances', async function () { + it('Should have the videos listed on the two instances', async function () { this.timeout(120000) await waitJobs(servers) for (const server of servers) { const res = await getVideosList(server.url) - expect(res.body.total).to.equal(2) - expect(res.body.data).to.have.lengthOf(2) + expect(res.body.total).to.equal(4) + expect(res.body.data).to.have.lengthOf(4) await checkVideoServer2(server.url, res.body.data[0].uuid) - await checkVideoServer1(server.url, res.body.data[1].uuid) + + const [ ,videoHttp, videoMagnet, videoTorrent ] = res.body.data + await checkVideosServer1(server.url, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid) + } + }) + + it('Should import a video that will be transcoded', async function () { + this.timeout(120000) + + const attributes = { + name: 'transcoded video', + magnetUri: getMagnetURI(), + channelId: channelIdServer2, + privacy: VideoPrivacy.PUBLIC + } + const res = await importVideo(servers[1].url, servers[1].accessToken, attributes) + const videoUUID = res.body.video.uuid + + await waitJobs(servers) + + for (const server of servers) { + const res = await getVideo(server.url, videoUUID) + const video: VideoDetails = res.body + + expect(video.name).to.equal('transcoded video') + expect(video.files).to.have.lengthOf(4) } }) diff --git a/server/tests/fixtures/60fps_small-240p.torrent b/server/tests/fixtures/60fps_small-240p.torrent deleted file mode 100644 index ec4c0babe..000000000 --- a/server/tests/fixtures/60fps_small-240p.torrent +++ /dev/null @@ -1 +0,0 @@ -d8:announce41:wss://peertube2.cpy.re:443/tracker/socket13:announce-listll41:wss://peertube2.cpy.re:443/tracker/socketel41:https://peertube2.cpy.re/tracker/announceee10:created by8:PeerTube13:creation datei1529593069e8:encoding5:UTF-84:infod6:lengthi30921e4:name20:60fps_small 240p.mp412:piece lengthi16384e6:pieces40:Ä–…+çéCFm7çc0ÏÅT-@2Ç6©0áMür|Rv›$˜h%e8:url-listl84:https://peertube2.cpy.re/static/webseed/2b8dbe74-9548-4f6f-a8da-986aed9e5e45-240.mp4ee \ No newline at end of file diff --git a/server/tests/fixtures/video-720p.torrent b/server/tests/fixtures/video-720p.torrent new file mode 100644 index 000000000..64bfd5220 Binary files /dev/null and b/server/tests/fixtures/video-720p.torrent differ diff --git a/server/tests/utils/videos/video-imports.ts b/server/tests/utils/videos/video-imports.ts index fa2f13b5e..59dfd481a 100644 --- a/server/tests/utils/videos/video-imports.ts +++ b/server/tests/utils/videos/video-imports.ts @@ -1,5 +1,5 @@ import { VideoImportCreate } from '../../../../shared/models/videos' -import { makeGetRequest, makePostBodyRequest, makeUploadRequest } from '..' +import { makeGetRequest, makeUploadRequest } from '..' function getYoutubeVideoUrl () { return 'https://youtu.be/msX3jv1XdvM' @@ -7,7 +7,7 @@ function getYoutubeVideoUrl () { function getMagnetURI () { // tslint:disable:max-line-length - return 'magnet:?xs=https%3A%2F%2Fpeertube2.cpy.re%2Fstatic%2Ftorrents%2F2b8dbe74-9548-4f6f-a8da-986aed9e5e45-240.torrent&xt=urn:btih:52bf3729e5859390a8751495196b5674a55c99f3&dn=60fps_small&tr=wss%3A%2F%2Fpeertube2.cpy.re%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fpeertube2.cpy.re%2Ftracker%2Fannounce&ws=https%3A%2F%2Fpeertube2.cpy.re%2Fstatic%2Fwebseed%2F2b8dbe74-9548-4f6f-a8da-986aed9e5e45-240.mp4' + return 'magnet:?xs=https%3A%2F%2Fpeertube2.cpy.re%2Fstatic%2Ftorrents%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.torrent&xt=urn:btih:0f498834733e8057ed5c6f2ee2b4efd8d84a76ee&dn=super+peertube2+video&tr=wss%3A%2F%2Fpeertube2.cpy.re%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fpeertube2.cpy.re%2Ftracker%2Fannounce&ws=https%3A%2F%2Fpeertube2.cpy.re%2Fstatic%2Fwebseed%2Fb209ca00-c8bb-4b2b-b421-1ede169f3dbc-720.mp4' } function importVideo (url: string, token: string, attributes: VideoImportCreate) { @@ -26,11 +26,15 @@ function importVideo (url: string, token: string, attributes: VideoImportCreate) }) } -function getMyVideoImports (url: string, token: string) { +function getMyVideoImports (url: string, token: string, sort?: string) { const path = '/api/v1/users/me/videos/imports' + const query = {} + if (sort) query['sort'] = sort + return makeGetRequest({ url, + query, path, token, statusCodeExpected: 200 -- cgit v1.2.3