From 3e17515e2996b79e23f569c296051a91af3fcbe4 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 7 Aug 2018 15:17:17 +0200 Subject: [PATCH] Add torrent tests --- server/controllers/api/videos/import.ts | 6 +- server/helpers/core-utils.ts | 5 +- server/helpers/utils.ts | 3 +- server/helpers/webtorrent.ts | 9 +- server/lib/job-queue/handlers/video-import.ts | 8 +- server/tests/api/check-params/users.ts | 2 +- .../tests/api/check-params/video-imports.ts | 2 +- server/tests/api/videos/video-imports.ts | 159 +++++++++++++----- .../tests/fixtures/60fps_small-240p.torrent | 1 - server/tests/fixtures/video-720p.torrent | Bin 0 -> 2644 bytes server/tests/utils/videos/video-imports.ts | 10 +- 11 files changed, 145 insertions(+), 60 deletions(-) delete mode 100644 server/tests/fixtures/60fps_small-240p.torrent create mode 100644 server/tests/fixtures/video-720p.torrent diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts index 94dafcdbd..b2f73fa48 100644 --- a/server/controllers/api/videos/import.ts +++ b/server/controllers/api/videos/import.ts @@ -97,7 +97,7 @@ async function addTorrentImport (req: express.Request, res: express.Response, to await processThumbnail(req, video) await processPreview(req, video) - const tags = null + const tags = body.tags || undefined const videoImportAttributes = { magnetUri, torrentName, @@ -224,11 +224,13 @@ function insertIntoDB ( videoCreated.VideoChannel = videoChannel // Set tags to the video - if (tags !== undefined) { + if (tags) { const tagInstances = await TagModel.findOrCreateTags(tags, t) await videoCreated.$set('Tags', tagInstances, sequelizeOptions) videoCreated.Tags = tagInstances + } else { + videoCreated.Tags = [] } // Create video import object in database diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 25eb6454a..3b38da66c 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts @@ -5,7 +5,7 @@ import * as bcrypt from 'bcrypt' import * as createTorrent from 'create-torrent' -import { pseudoRandomBytes } from 'crypto' +import { createHash, pseudoRandomBytes } from 'crypto' import { copyFile, readdir, readFile, rename, stat, Stats, unlink, writeFile } from 'fs' import * as mkdirp from 'mkdirp' import { isAbsolute, join } from 'path' @@ -13,7 +13,6 @@ import * as pem from 'pem' import * as rimraf from 'rimraf' import { URL } from 'url' import { truncate } from 'lodash' -import * as crypto from 'crypto' function sanitizeUrl (url: string) { const urlObject = new URL(url) @@ -97,7 +96,7 @@ function peertubeTruncate (str: string, maxLength: number) { } function sha256 (str: string) { - return crypto.createHash('sha256').update(str).digest('hex') + return createHash('sha256').update(str).digest('hex') } function promisify0 (func: (cb: (err: any, result: A) => void) => void): () => Promise { diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index 2ad87951e..eaad55555 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts @@ -9,8 +9,7 @@ import { ApplicationModel } from '../models/application/application' import { pseudoRandomBytesPromise, sha256, unlinkPromise } from './core-utils' import { logger } from './logger' import { isArray } from './custom-validators/misc' -import * as crypto from "crypto" -import { join } from "path" +import { join } from 'path' import { Instance as ParseTorrent } from 'parse-torrent' const isCidr = require('is-cidr') diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index 04b3ac71b..121cd0b41 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -2,7 +2,6 @@ import { logger } from './logger' import { generateVideoTmpPath } from './utils' import * as WebTorrent from 'webtorrent' import { createWriteStream } from 'fs' -import { Instance as ParseTorrent } from 'parse-torrent' import { CONFIG } from '../initializers' import { join } from 'path' @@ -20,10 +19,12 @@ function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: stri if (torrent.files.length !== 1) return rej(new Error('The number of files is not equal to 1 for ' + torrentId)) const file = torrent.files[ 0 ] - file.createReadStream().pipe(createWriteStream(path)) - }) - torrent.on('done', () => res(path)) + const writeStream = createWriteStream(path) + writeStream.on('finish', () => res(path)) + + file.createReadStream().pipe(writeStream) + }) torrent.on('error', err => rej(err)) }) diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index 28a03d19e..2d19b82a4 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts @@ -114,8 +114,8 @@ async function processFile (downloader: () => Promise, videoImport: Vide tempVideoPath = await downloader() // Get information about this video - const { size } = await statPromise(tempVideoPath) - const isAble = await videoImport.User.isAbleToUploadVideo({ size }) + const stats = await statPromise(tempVideoPath) + const isAble = await videoImport.User.isAbleToUploadVideo({ size: stats.size }) if (isAble === false) { throw new Error('The user video quota is exceeded with this video to import.') } @@ -128,7 +128,7 @@ async function processFile (downloader: () => Promise, videoImport: Vide const videoFileData = { extname: extname(tempVideoPath), resolution: videoFileResolution, - size, + size: stats.size, fps, videoId: videoImport.videoId } @@ -209,7 +209,7 @@ async function processFile (downloader: () => Promise, videoImport: Vide } catch (err) { try { - if (tempVideoPath) await unlinkPromise(tempVideoPath) + // if (tempVideoPath) await unlinkPromise(tempVideoPath) } catch (errUnlink) { logger.warn('Cannot cleanup files after a video import error.', { err: errUnlink }) } diff --git a/server/tests/api/check-params/users.ts b/server/tests/api/check-params/users.ts index e1655e85b..7b25df29f 100644 --- a/server/tests/api/check-params/users.ts +++ b/server/tests/api/check-params/users.ts @@ -627,7 +627,7 @@ describe('Test users API validators', function () { } await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { targetUrl: getYoutubeVideoUrl() })) await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { magnetUri: getMagnetURI() })) - await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { torrentfile: '60fps_small-240p.torrent' })) + await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { torrentfile: 'video-720p.torrent' })) await waitJobs([ server ]) diff --git a/server/tests/api/check-params/video-imports.ts b/server/tests/api/check-params/video-imports.ts index e62f0918e..38ddd4e56 100644 --- a/server/tests/api/check-params/video-imports.ts +++ b/server/tests/api/check-params/video-imports.ts @@ -303,7 +303,7 @@ describe('Test video imports API validator', function () { fields = omit(fields, 'magnetUri') const attaches = { - 'torrentfile': join(__dirname, '..', '..', 'fixtures', '60fps_small-240p.torrent') + 'torrentfile': join(__dirname, '..', '..', 'fixtures', 'video-720p.torrent') } await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches, statusCodeExpected: 409 }) diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts index f21ade5c3..b7866d529 100644 --- a/server/tests/api/videos/video-imports.ts +++ b/server/tests/api/videos/video-imports.ts @@ -2,7 +2,7 @@ import * as chai from 'chai' import 'mocha' -import { VideoDetails, VideoPrivacy } from '../../../../shared/models/videos' +import { VideoDetails, VideoImport, VideoPrivacy } from '../../../../shared/models/videos' import { doubleFollow, flushAndRunMultipleServers, @@ -10,12 +10,13 @@ import { getMyVideos, getVideo, getVideosList, + immutableAssign, killallServers, ServerInfo, setAccessTokensToServers } from '../../utils' import { waitJobs } from '../../utils/server/jobs' -import { getMyVideoImports, getYoutubeVideoUrl, importVideo } from '../../utils/videos/video-imports' +import { getMagnetURI, getYoutubeVideoUrl, importVideo, getMyVideoImports } from '../../utils/videos/video-imports' const expect = chai.expect @@ -24,19 +25,36 @@ describe('Test video imports', function () { let channelIdServer1: number let channelIdServer2: number - async function checkVideoServer1 (url: string, id: number | string) { - const res = await getVideo(url, id) - const video: VideoDetails = res.body - - expect(video.name).to.equal('small video - youtube') - expect(video.category.label).to.equal('News') - expect(video.licence.label).to.equal('Attribution') - expect(video.language.label).to.equal('Unknown') - expect(video.nsfw).to.be.false - expect(video.description).to.equal('this is a super description') - expect(video.tags).to.deep.equal([ 'tag1', 'tag2' ]) + async function checkVideosServer1 (url: string, idHttp: string, idMagnet: string, idTorrent: string) { + const resHttp = await getVideo(url, idHttp) + const videoHttp: VideoDetails = resHttp.body + + expect(videoHttp.name).to.equal('small video - youtube') + expect(videoHttp.category.label).to.equal('News') + expect(videoHttp.licence.label).to.equal('Attribution') + expect(videoHttp.language.label).to.equal('Unknown') + expect(videoHttp.nsfw).to.be.false + expect(videoHttp.description).to.equal('this is a super description') + expect(videoHttp.tags).to.deep.equal([ 'tag1', 'tag2' ]) + expect(videoHttp.files).to.have.lengthOf(1) + + const resMagnet = await getVideo(url, idMagnet) + const videoMagnet: VideoDetails = resMagnet.body + const resTorrent = await getVideo(url, idTorrent) + const videoTorrent: VideoDetails = resTorrent.body + + for (const video of [ videoMagnet, videoTorrent ]) { + expect(video.category.label).to.equal('Misc') + expect(video.licence.label).to.equal('Unknown') + expect(video.language.label).to.equal('Unknown') + expect(video.nsfw).to.be.false + expect(video.description).to.equal('this is a super torrent description') + expect(video.tags).to.deep.equal([ 'tag_torrent1', 'tag_torrent2' ]) + expect(video.files).to.have.lengthOf(1) + } - expect(video.files).to.have.lengthOf(1) + expect(videoTorrent.name).to.contain('你好 世界 720p.mp4') + expect(videoMagnet.name).to.contain('super peertube2 video') } async function checkVideoServer2 (url: string, id: number | string) { @@ -75,50 +93,88 @@ describe('Test video imports', function () { await doubleFollow(servers[0], servers[1]) }) - it('Should import a video on server 1', async function () { + it('Should import videos on server 1', async function () { this.timeout(60000) - const attributes = { - targetUrl: getYoutubeVideoUrl(), + const baseAttributes = { channelId: channelIdServer1, privacy: VideoPrivacy.PUBLIC } - const res = await importVideo(servers[0].url, servers[0].accessToken, attributes) - expect(res.body.video.name).to.equal('small video - youtube') + + { + const attributes = immutableAssign(baseAttributes, { targetUrl: getYoutubeVideoUrl() }) + const res = await importVideo(servers[0].url, servers[0].accessToken, attributes) + expect(res.body.video.name).to.equal('small video - youtube') + } + + { + const attributes = immutableAssign(baseAttributes, { + magnetUri: getMagnetURI(), + description: 'this is a super torrent description', + tags: [ 'tag_torrent1', 'tag_torrent2' ] + }) + const res = await importVideo(servers[0].url, servers[0].accessToken, attributes) + expect(res.body.video.name).to.equal('super peertube2 video') + } + + { + const attributes = immutableAssign(baseAttributes, { + torrentfile: 'video-720p.torrent', + description: 'this is a super torrent description', + tags: [ 'tag_torrent1', 'tag_torrent2' ] + }) + const res = await importVideo(servers[0].url, servers[0].accessToken, attributes) + expect(res.body.video.name).to.equal('你好 世界 720p.mp4') + } }) - it('Should list the video to import in my videos on server 1', async function () { - const res = await getMyVideos(servers[0].url, servers[0].accessToken, 0, 5) + it('Should list the videos to import in my videos on server 1', async function () { + const res = await getMyVideos(servers[0].url, servers[0].accessToken, 0, 5, 'createdAt') - expect(res.body.total).to.equal(1) + expect(res.body.total).to.equal(3) const videos = res.body.data - expect(videos).to.have.lengthOf(1) + expect(videos).to.have.lengthOf(3) expect(videos[0].name).to.equal('small video - youtube') + expect(videos[1].name).to.equal('super peertube2 video') + expect(videos[2].name).to.equal('你好 世界 720p.mp4') }) - it('Should list the video to import in my imports on server 1', async function () { - const res = await getMyVideoImports(servers[0].url, servers[0].accessToken) + it('Should list the videos to import in my imports on server 1', async function () { + const res = await getMyVideoImports(servers[0].url, servers[0].accessToken, '-createdAt') + + expect(res.body.total).to.equal(3) + const videoImports: VideoImport[] = res.body.data + expect(videoImports).to.have.lengthOf(3) + + expect(videoImports[2].targetUrl).to.equal(getYoutubeVideoUrl()) + expect(videoImports[2].magnetUri).to.be.null + expect(videoImports[2].torrentName).to.be.null + expect(videoImports[2].video.name).to.equal('small video - youtube') - expect(res.body.total).to.equal(1) - const videoImports = res.body.data - expect(videoImports).to.have.lengthOf(1) + expect(videoImports[1].targetUrl).to.be.null + expect(videoImports[1].magnetUri).to.equal(getMagnetURI()) + expect(videoImports[1].torrentName).to.be.null + expect(videoImports[1].video.name).to.equal('super peertube2 video') - expect(videoImports[0].targetUrl).to.equal(getYoutubeVideoUrl()) - expect(videoImports[0].video.name).to.equal('small video - youtube') + expect(videoImports[0].targetUrl).to.be.null + expect(videoImports[0].magnetUri).to.be.null + expect(videoImports[0].torrentName).to.equal('video-720p.torrent') + expect(videoImports[0].video.name).to.equal('你好 世界 720p.mp4') }) - it('Should have the video listed on the two instances1', async function () { + it('Should have the video listed on the two instances', async function () { this.timeout(120000) await waitJobs(servers) for (const server of servers) { const res = await getVideosList(server.url) - expect(res.body.total).to.equal(1) - expect(res.body.data).to.have.lengthOf(1) + expect(res.body.total).to.equal(3) + expect(res.body.data).to.have.lengthOf(3) - await checkVideoServer1(server.url, res.body.data[0].uuid) + const [ videoHttp, videoMagnet, videoTorrent ] = res.body.data + await checkVideosServer1(server.url, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid) } }) @@ -127,7 +183,7 @@ describe('Test video imports', function () { const attributes = { targetUrl: getYoutubeVideoUrl(), - channelId: channelIdServer1, + channelId: channelIdServer2, privacy: VideoPrivacy.PUBLIC, category: 10, licence: 7, @@ -140,18 +196,43 @@ describe('Test video imports', function () { expect(res.body.video.name).to.equal('my super name') }) - it('Should have the video listed on the two instances', async function () { + it('Should have the videos listed on the two instances', async function () { this.timeout(120000) await waitJobs(servers) for (const server of servers) { const res = await getVideosList(server.url) - expect(res.body.total).to.equal(2) - expect(res.body.data).to.have.lengthOf(2) + expect(res.body.total).to.equal(4) + expect(res.body.data).to.have.lengthOf(4) await checkVideoServer2(server.url, res.body.data[0].uuid) - await checkVideoServer1(server.url, res.body.data[1].uuid) + + const [ ,videoHttp, videoMagnet, videoTorrent ] = res.body.data + await checkVideosServer1(server.url, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid) + } + }) + + it('Should import a video that will be transcoded', async function () { + this.timeout(120000) + + const attributes = { + name: 'transcoded video', + magnetUri: getMagnetURI(), + channelId: channelIdServer2, + privacy: VideoPrivacy.PUBLIC + } + const res = await importVideo(servers[1].url, servers[1].accessToken, attributes) + const videoUUID = res.body.video.uuid + + await waitJobs(servers) + + for (const server of servers) { + const res = await getVideo(server.url, videoUUID) + const video: VideoDetails = res.body + + expect(video.name).to.equal('transcoded video') + expect(video.files).to.have.lengthOf(4) } }) diff --git a/server/tests/fixtures/60fps_small-240p.torrent b/server/tests/fixtures/60fps_small-240p.torrent deleted file mode 100644 index ec4c0babe..000000000 --- a/server/tests/fixtures/60fps_small-240p.torrent +++ /dev/null @@ -1 +0,0 @@ -d8:announce41:wss://peertube2.cpy.re:443/tracker/socket13:announce-listll41:wss://peertube2.cpy.re:443/tracker/socketel41:https://peertube2.cpy.re/tracker/announceee10:created by8:PeerTube13:creation datei1529593069e8:encoding5:UTF-84:infod6:lengthi30921e4:name20:60fps_small 240p.mp412:piece lengthi16384e6:pieces40:Ė…+çéCFm7çc0ÏÅT-@2Ç6©0áMür|Rv›$˜h%e8:url-listl84:https://peertube2.cpy.re/static/webseed/2b8dbe74-9548-4f6f-a8da-986aed9e5e45-240.mp4ee \ No newline at end of file diff --git a/server/tests/fixtures/video-720p.torrent b/server/tests/fixtures/video-720p.torrent new file mode 100644 index 0000000000000000000000000000000000000000..64bfd5220c68adef67006776e653057fb82771d9 GIT binary patch literal 2644 zcmb7`X*d)L8-`^t6+&5(HDx;)vto>Wmz}KPl36U7F`5|^S&E7hne2OHUngs1kE~gu zLdh;+vM-@I&UdbJetqAM@7H@>&-*^t^Za;!+*l;a)!Uos>y5!7peTPb83hLW;BX|0 zFB%7v$N1clC*e>C1RP8uxnjI*ZbYmi3Xk)4r+9cmkx+;N42eUayj=-67z{Pnm^GiShLj?OYfP*^@-D7S{Vu@KQ41SI{hm%3 z^3JpkZ2(Uv$!yWIpX+A6GA9WC@^zxIdU2V0<%d-!xxfaLw&hpqk)%zKat+|nGVCqp zq?#0v{5pb8ZU3IE#+x@AcvU)_YVvL*e(vJhy3cy@8n5sMARf8jA-BG@8V>V7# zL1p>%PEM>2CoTk(bw)-?dCBZlSw)WuKAwpZCvbm#bzkIUZ?)k@e8{Z8mycZZT6HA5 z%H=1%mjo?bwq`^EauN?fpMVC`JYwK49?e3@h^PlGwLp7?kv=12w(cy6FIqe!$RMWr z1Gim3_%idJT5blpXnH+fIsc=_dWtz;$H8XetP-|6$Yk*m3o~1~<-w&EQ!c{kDgNv5 zIwm+Rd2!aK35fOFJM%NwRqX!Qxhjt#2LCy8ze@f+Z9*IJ6t6{B68>dz(Zh=tnIn&& z)`V}#mTqUeEritO@}_^W)y*xkv!7#oQWMWZXijIlxoH>3AC?mm3%d(DYZmLQbC<4h z8r^cEmdnNG(!%#XAiQ!dT1)QLjMtllmVK7@)w8sBmO7nU&svVwdR28jv*>B20Rm=% zFI0a=T|-- zt;4^5xSuW8v6Oi&DVpnjcS+S~W`FY3hDQ9X^2Wnk7Y>0rffs!siV~G9eB&HRj*+v! z=eug;C^zYd!kC`|*1|ePKN?T;2nuyrkc1f>%#ERM0XK6D-WOz+P|0b(PcY;qjSN?b z(%Tx6lkOU(oknlUYMe!0El?NFY6g+$!%ky=TwkOUt;m*p@bGwZaMzgQcIU5agGe*N{my)iO@1LRdCd7!RoO?s5BF?c z8KY0Vsvlvt>2g5GXb8Mab^cU_lUPh}x!#LkEa{}K^xfOzt)3HUihaTh&bnoqy+JJO zyGC?%KMUqdIpWkpJiYfF@NS|(l9Rhb)ST`j~ ziOT?m)TvNTD*E}xbn1o<&xDJJYS;1=wyb>@4Qwo@LR1%$w1}5*kNMVpcbSJmLPvJZ zVDstvtr9!K&S#F=)b0EA8H&(BK?A0e!?8xSUGwB@;4dPw_=9>_TPV*b%`uww&hy$8 z>3mDe;gwz2YTrB_`^?}Y{id;He#4?t_O!~+$!)Cj+GeybjJGLC#_s6Qh z$I6twL|H95yO{a}$TGT#slA6BNZ=MZ={LdoInFrJ@!7Rbp)%G!3%#_a%lL{pKK2k; zcX`zrOuJ~;8Z;|$C z4m*WH4+*#jw=qb0TwPJ~EWc!;UJM-U-#Zsm%^G|DGSK8aTOn>ch&-*Pqd^tyLpCD`^?5^%xzW2wRj&}HkZ1S5*yET^d# zkTywLkv_^b*RfFBDHL;u*U22@hO5r_(wU3D4CJ`^A;X0skFFD|&*zc>)@McCR#oycFQ3D1@Uu)ZyRk~521cpQ2#L@tQd1sKBC;l9Xg)s znD}F>aR3eZ=2jH?)Sq+ou-xD&U_Eo0?@9GtHszqId!sY%GmCROc1iEdYqz>sqZL~i z_NQCF)9Na|oS-_e$EX{2ss=_th97Hx8~xC(v{Aa$WSeCj7R#D-E00~w9N?C43J)om z_CLL0B}SA>L77PHcnyw*Z?dpa$8DIk(}UwhFMZsqh-Hnx$Fea{LLhARO&3s3s4yQ= z14X|V<_7rpUnw2xEw2