X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Ftests%2Fapi%2Fvideos%2Fvideo-imports.ts;h=e8e0f01f1838088b7ea19ee986aa5a4fedcd4224;hb=c55e3d7227fe1453869e309025996b9d75256d5d;hp=aaee79a4abab1351ab9fa950d29dafc652644e32;hpb=bb5d90e62f631af3c899fbe586485e64938a5927;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts index aaee79a4a..e8e0f01f1 100644 --- a/server/tests/api/videos/video-imports.ts +++ b/server/tests/api/videos/video-imports.ts @@ -1,242 +1,456 @@ -/* tslint:disable:no-unused-expression */ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import * as chai from 'chai' import 'mocha' -import { VideoDetails, VideoImport, VideoPrivacy } from '../../../../shared/models/videos' +import { expect } from 'chai' +import { pathExists, readdir, remove } from 'fs-extra' +import { join } from 'path' +import { FIXTURE_URLS, testCaptionFile, testImage } from '@server/tests/shared' +import { areHttpImportTestsDisabled } from '@shared/core-utils' +import { VideoPrivacy, VideoResolution } from '@shared/models' import { + cleanupTests, + createMultipleServers, + createSingleServer, doubleFollow, - flushAndRunMultipleServers, - getMyUserInformation, - getMyVideos, - getVideo, - getVideosList, - immutableAssign, - killallServers, - ServerInfo, - setAccessTokensToServers -} from '../../utils' -import { waitJobs } from '../../utils/server/jobs' -import { getMagnetURI, getYoutubeVideoUrl, importVideo, getMyVideoImports } from '../../utils/videos/video-imports' - -const expect = chai.expect + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs +} from '@shared/server-commands' + +async function checkVideosServer1 (server: PeerTubeServer, idHttp: string, idMagnet: string, idTorrent: string) { + const videoHttp = await server.videos.get({ id: idHttp }) + + expect(videoHttp.name).to.equal('small video - youtube') + // FIXME: youtube-dl seems broken + // expect(videoHttp.category.label).to.equal('News & Politics') + // expect(videoHttp.licence.label).to.equal('Attribution') + expect(videoHttp.language.label).to.equal('Unknown') + expect(videoHttp.nsfw).to.be.false + expect(videoHttp.description).to.equal('this is a super description') + expect(videoHttp.tags).to.deep.equal([ 'tag1', 'tag2' ]) + expect(videoHttp.files).to.have.lengthOf(1) + + const originallyPublishedAt = new Date(videoHttp.originallyPublishedAt) + expect(originallyPublishedAt.getDate()).to.equal(14) + expect(originallyPublishedAt.getMonth()).to.equal(0) + expect(originallyPublishedAt.getFullYear()).to.equal(2019) + + const videoMagnet = await server.videos.get({ id: idMagnet }) + const videoTorrent = await server.videos.get({ id: idTorrent }) + + for (const video of [ videoMagnet, videoTorrent ]) { + expect(video.category.label).to.equal('Misc') + expect(video.licence.label).to.equal('Unknown') + expect(video.language.label).to.equal('Unknown') + expect(video.nsfw).to.be.false + expect(video.description).to.equal('this is a super torrent description') + expect(video.tags).to.deep.equal([ 'tag_torrent1', 'tag_torrent2' ]) + expect(video.files).to.have.lengthOf(1) + } + + expect(videoTorrent.name).to.contain('你好 世界 720p.mp4') + expect(videoMagnet.name).to.contain('super peertube2 video') + + const bodyCaptions = await server.captions.list({ videoId: idHttp }) + expect(bodyCaptions.total).to.equal(2) +} + +async function checkVideoServer2 (server: PeerTubeServer, id: number | string) { + const video = await server.videos.get({ id }) + + expect(video.name).to.equal('my super name') + expect(video.category.label).to.equal('Entertainment') + expect(video.licence.label).to.equal('Public Domain Dedication') + expect(video.language.label).to.equal('English') + expect(video.nsfw).to.be.false + expect(video.description).to.equal('my super description') + expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ]) + + expect(video.files).to.have.lengthOf(1) + + const bodyCaptions = await server.captions.list({ videoId: id }) + expect(bodyCaptions.total).to.equal(2) +} describe('Test video imports', function () { - let servers: ServerInfo[] = [] - let channelIdServer1: number - let channelIdServer2: number - - async function checkVideosServer1 (url: string, idHttp: string, idMagnet: string, idTorrent: string) { - const resHttp = await getVideo(url, idHttp) - const videoHttp: VideoDetails = resHttp.body - - expect(videoHttp.name).to.equal('small video - youtube') - expect(videoHttp.category.label).to.equal('News & Politics') - expect(videoHttp.licence.label).to.equal('Attribution') - expect(videoHttp.language.label).to.equal('Unknown') - expect(videoHttp.nsfw).to.be.false - expect(videoHttp.description).to.equal('this is a super description') - expect(videoHttp.tags).to.deep.equal([ 'tag1', 'tag2' ]) - expect(videoHttp.files).to.have.lengthOf(1) - - const resMagnet = await getVideo(url, idMagnet) - const videoMagnet: VideoDetails = resMagnet.body - const resTorrent = await getVideo(url, idTorrent) - const videoTorrent: VideoDetails = resTorrent.body - - for (const video of [ videoMagnet, videoTorrent ]) { - expect(video.category.label).to.equal('Misc') - expect(video.licence.label).to.equal('Unknown') - expect(video.language.label).to.equal('Unknown') - expect(video.nsfw).to.be.false - expect(video.description).to.equal('this is a super torrent description') - expect(video.tags).to.deep.equal([ 'tag_torrent1', 'tag_torrent2' ]) - expect(video.files).to.have.lengthOf(1) - } - expect(videoTorrent.name).to.contain('你好 世界 720p.mp4') - expect(videoMagnet.name).to.contain('super peertube2 video') - } + if (areHttpImportTestsDisabled()) return - async function checkVideoServer2 (url: string, id: number | string) { - const res = await getVideo(url, id) - const video = res.body + function runSuite (mode: 'youtube-dl' | 'yt-dlp') { - expect(video.name).to.equal('my super name') - expect(video.category.label).to.equal('Entertainment') - expect(video.licence.label).to.equal('Public Domain Dedication') - expect(video.language.label).to.equal('English') - expect(video.nsfw).to.be.false - expect(video.description).to.equal('my super description') - expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ]) + describe('Import ' + mode, function () { + let servers: PeerTubeServer[] = [] - expect(video.files).to.have.lengthOf(1) - } + before(async function () { + this.timeout(30_000) - before(async function () { - this.timeout(30000) + // Run servers + servers = await createMultipleServers(2, { + import: { + videos: { + http: { + youtube_dl_release: { + url: mode === 'youtube-dl' + ? 'https://yt-dl.org/downloads/latest/youtube-dl' + : 'https://api.github.com/repos/yt-dlp/yt-dlp/releases', - // Run servers - servers = await flushAndRunMultipleServers(2) + name: mode + } + } + } + } + }) - await setAccessTokensToServers(servers) + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) - { - const res = await getMyUserInformation(servers[0].url, servers[0].accessToken) - channelIdServer1 = res.body.videoChannels[ 0 ].id - } + await doubleFollow(servers[0], servers[1]) + }) - { - const res = await getMyUserInformation(servers[1].url, servers[1].accessToken) - channelIdServer2 = res.body.videoChannels[ 0 ].id - } + it('Should import videos on server 1', async function () { + this.timeout(60_000) + + const baseAttributes = { + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + + { + const attributes = { ...baseAttributes, targetUrl: FIXTURE_URLS.youtube } + const { video } = await servers[0].imports.importVideo({ attributes }) + expect(video.name).to.equal('small video - youtube') + + { + expect(video.thumbnailPath).to.match(new RegExp(`^/static/thumbnails/.+.jpg$`)) + expect(video.previewPath).to.match(new RegExp(`^/lazy-static/previews/.+.jpg$`)) + + const suffix = mode === 'yt-dlp' + ? '_yt_dlp' + : '' + + await testImage(servers[0].url, 'video_import_thumbnail' + suffix, video.thumbnailPath) + await testImage(servers[0].url, 'video_import_preview' + suffix, video.previewPath) + } + + const bodyCaptions = await servers[0].captions.list({ videoId: video.id }) + const videoCaptions = bodyCaptions.data + expect(videoCaptions).to.have.lengthOf(2) + + { + const enCaption = videoCaptions.find(caption => caption.language.id === 'en') + expect(enCaption).to.exist + expect(enCaption.language.label).to.equal('English') + expect(enCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-en.vtt$`)) + + const regex = `WEBVTT[ \n]+Kind: captions[ \n]+Language: en[ \n]+00:00:01.600 --> 00:00:04.200[ \n]+English \\(US\\)[ \n]+` + + `00:00:05.900 --> 00:00:07.999[ \n]+This is a subtitle in American English[ \n]+` + + `00:00:10.000 --> 00:00:14.000[ \n]+Adding subtitles is very easy to do` + await testCaptionFile(servers[0].url, enCaption.captionPath, new RegExp(regex)) + } + + { + const frCaption = videoCaptions.find(caption => caption.language.id === 'fr') + expect(frCaption).to.exist + expect(frCaption.language.label).to.equal('French') + expect(frCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-fr.vtt`)) + + const regex = `WEBVTT[ \n]+Kind: captions[ \n]+Language: fr[ \n]+00:00:01.600 --> 00:00:04.200[ \n]+` + + `Français \\(FR\\)[ \n]+00:00:05.900 --> 00:00:07.999[ \n]+C'est un sous-titre français[ \n]+` + + `00:00:10.000 --> 00:00:14.000[ \n]+Ajouter un sous-titre est vraiment facile` + + await testCaptionFile(servers[0].url, frCaption.captionPath, new RegExp(regex)) + } + } + + { + const attributes = { + ...baseAttributes, + magnetUri: FIXTURE_URLS.magnet, + description: 'this is a super torrent description', + tags: [ 'tag_torrent1', 'tag_torrent2' ] + } + const { video } = await servers[0].imports.importVideo({ attributes }) + expect(video.name).to.equal('super peertube2 video') + } + + { + const attributes = { + ...baseAttributes, + torrentfile: 'video-720p.torrent' as any, + description: 'this is a super torrent description', + tags: [ 'tag_torrent1', 'tag_torrent2' ] + } + const { video } = await servers[0].imports.importVideo({ attributes }) + expect(video.name).to.equal('你好 世界 720p.mp4') + } + }) - await doubleFollow(servers[0], servers[1]) - }) + it('Should list the videos to import in my videos on server 1', async function () { + const { total, data } = await servers[0].videos.listMyVideos({ sort: 'createdAt' }) - it('Should import videos on server 1', async function () { - this.timeout(60000) + expect(total).to.equal(3) - const baseAttributes = { - channelId: channelIdServer1, - privacy: VideoPrivacy.PUBLIC - } + expect(data).to.have.lengthOf(3) + expect(data[0].name).to.equal('small video - youtube') + expect(data[1].name).to.equal('super peertube2 video') + expect(data[2].name).to.equal('你好 世界 720p.mp4') + }) - { - const attributes = immutableAssign(baseAttributes, { targetUrl: getYoutubeVideoUrl() }) - const res = await importVideo(servers[0].url, servers[0].accessToken, attributes) - expect(res.body.video.name).to.equal('small video - youtube') - } + it('Should list the videos to import in my imports on server 1', async function () { + const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ sort: '-createdAt' }) + expect(total).to.equal(3) + + expect(videoImports).to.have.lengthOf(3) + + expect(videoImports[2].targetUrl).to.equal(FIXTURE_URLS.youtube) + expect(videoImports[2].magnetUri).to.be.null + expect(videoImports[2].torrentName).to.be.null + expect(videoImports[2].video.name).to.equal('small video - youtube') - { - const attributes = immutableAssign(baseAttributes, { - magnetUri: getMagnetURI(), - description: 'this is a super torrent description', - tags: [ 'tag_torrent1', 'tag_torrent2' ] + expect(videoImports[1].targetUrl).to.be.null + expect(videoImports[1].magnetUri).to.equal(FIXTURE_URLS.magnet) + expect(videoImports[1].torrentName).to.be.null + expect(videoImports[1].video.name).to.equal('super peertube2 video') + + expect(videoImports[0].targetUrl).to.be.null + expect(videoImports[0].magnetUri).to.be.null + expect(videoImports[0].torrentName).to.equal('video-720p.torrent') + expect(videoImports[0].video.name).to.equal('你好 世界 720p.mp4') }) - const res = await importVideo(servers[0].url, servers[0].accessToken, attributes) - expect(res.body.video.name).to.equal('super peertube2 video') - } - { - const attributes = immutableAssign(baseAttributes, { - torrentfile: 'video-720p.torrent', - description: 'this is a super torrent description', - tags: [ 'tag_torrent1', 'tag_torrent2' ] + it('Should have the video listed on the two instances', async function () { + this.timeout(120_000) + + await waitJobs(servers) + + for (const server of servers) { + const { total, data } = await server.videos.list() + expect(total).to.equal(3) + expect(data).to.have.lengthOf(3) + + const [ videoHttp, videoMagnet, videoTorrent ] = data + await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid) + } + }) + + it('Should import a video on server 2 with some fields', async function () { + this.timeout(60_000) + + const attributes = { + targetUrl: FIXTURE_URLS.youtube, + channelId: servers[1].store.channel.id, + privacy: VideoPrivacy.PUBLIC, + category: 10, + licence: 7, + language: 'en', + name: 'my super name', + description: 'my super description', + tags: [ 'supertag1', 'supertag2' ] + } + const { video } = await servers[1].imports.importVideo({ attributes }) + expect(video.name).to.equal('my super name') }) - const res = await importVideo(servers[0].url, servers[0].accessToken, attributes) - expect(res.body.video.name).to.equal('你好 世界 720p.mp4') - } - }) - it('Should list the videos to import in my videos on server 1', async function () { - const res = await getMyVideos(servers[0].url, servers[0].accessToken, 0, 5, 'createdAt') + it('Should have the videos listed on the two instances', async function () { + this.timeout(120_000) - expect(res.body.total).to.equal(3) + await waitJobs(servers) - const videos = res.body.data - expect(videos).to.have.lengthOf(3) - expect(videos[0].name).to.equal('small video - youtube') - expect(videos[1].name).to.equal('super peertube2 video') - expect(videos[2].name).to.equal('你好 世界 720p.mp4') - }) + for (const server of servers) { + const { total, data } = await server.videos.list() + expect(total).to.equal(4) + expect(data).to.have.lengthOf(4) - it('Should list the videos to import in my imports on server 1', async function () { - const res = await getMyVideoImports(servers[0].url, servers[0].accessToken, '-createdAt') + await checkVideoServer2(server, data[0].uuid) - expect(res.body.total).to.equal(3) - const videoImports: VideoImport[] = res.body.data - expect(videoImports).to.have.lengthOf(3) + const [ , videoHttp, videoMagnet, videoTorrent ] = data + await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid) + } + }) - expect(videoImports[2].targetUrl).to.equal(getYoutubeVideoUrl()) - expect(videoImports[2].magnetUri).to.be.null - expect(videoImports[2].torrentName).to.be.null - expect(videoImports[2].video.name).to.equal('small video - youtube') + it('Should import a video that will be transcoded', async function () { + this.timeout(240_000) - expect(videoImports[1].targetUrl).to.be.null - expect(videoImports[1].magnetUri).to.equal(getMagnetURI()) - expect(videoImports[1].torrentName).to.be.null - expect(videoImports[1].video.name).to.equal('super peertube2 video') + const attributes = { + name: 'transcoded video', + magnetUri: FIXTURE_URLS.magnet, + channelId: servers[1].store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + const { video } = await servers[1].imports.importVideo({ attributes }) + const videoUUID = video.uuid - expect(videoImports[0].targetUrl).to.be.null - expect(videoImports[0].magnetUri).to.be.null - expect(videoImports[0].torrentName).to.equal('video-720p.torrent') - expect(videoImports[0].video.name).to.equal('你好 世界 720p.mp4') - }) + await waitJobs(servers) - it('Should have the video listed on the two instances', async function () { - this.timeout(120000) + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) - await waitJobs(servers) + expect(video.name).to.equal('transcoded video') + expect(video.files).to.have.lengthOf(4) + } + }) - for (const server of servers) { - const res = await getVideosList(server.url) - expect(res.body.total).to.equal(3) - expect(res.body.data).to.have.lengthOf(3) + it('Should import no HDR version on a HDR video', async function () { + this.timeout(300_000) + + const config = { + transcoding: { + enabled: true, + resolutions: { + '144p': true, + '240p': true, + '360p': false, + '480p': false, + '720p': false, + '1080p': false, // the resulting resolution shouldn't be higher than this, and not vp9.2/av01 + '1440p': false, + '2160p': false + }, + webtorrent: { enabled: true }, + hls: { enabled: false } + }, + import: { + videos: { + http: { + enabled: true + }, + torrent: { + enabled: true + } + } + } + } + await servers[0].config.updateCustomSubConfig({ newConfig: config }) + + const attributes = { + name: 'hdr video', + targetUrl: FIXTURE_URLS.youtubeHDR, + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + const { video: videoImported } = await servers[0].imports.importVideo({ attributes }) + const videoUUID = videoImported.uuid + + await waitJobs(servers) + + // test resolution + const video = await servers[0].videos.get({ id: videoUUID }) + expect(video.name).to.equal('hdr video') + const maxResolution = Math.max.apply(Math, video.files.map(function (o) { return o.resolution.id })) + expect(maxResolution, 'expected max resolution not met').to.equals(VideoResolution.H_240P) + }) - const [ videoHttp, videoMagnet, videoTorrent ] = res.body.data - await checkVideosServer1(server.url, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid) - } - }) + it('Should import a peertube video', async function () { + this.timeout(120_000) - it('Should import a video on server 2 with some fields', async function () { - this.timeout(60000) - - const attributes = { - targetUrl: getYoutubeVideoUrl(), - channelId: channelIdServer2, - privacy: VideoPrivacy.PUBLIC, - category: 10, - licence: 7, - language: 'en', - name: 'my super name', - description: 'my super description', - tags: [ 'supertag1', 'supertag2' ] - } - const res = await importVideo(servers[1].url, servers[1].accessToken, attributes) - expect(res.body.video.name).to.equal('my super name') - }) + const toTest = [ FIXTURE_URLS.peertube_long ] - it('Should have the videos listed on the two instances', async function () { - this.timeout(120000) + // TODO: include peertube_short when https://github.com/ytdl-org/youtube-dl/pull/29475 is merged + if (mode === 'yt-dlp') { + toTest.push(FIXTURE_URLS.peertube_short) + } - await waitJobs(servers) + for (const targetUrl of toTest) { + await servers[0].config.disableTranscoding() - for (const server of servers) { - const res = await getVideosList(server.url) - expect(res.body.total).to.equal(4) - expect(res.body.data).to.have.lengthOf(4) + const attributes = { + targetUrl, + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + const { video } = await servers[0].imports.importVideo({ attributes }) + const videoUUID = video.uuid - await checkVideoServer2(server.url, res.body.data[0].uuid) + await waitJobs(servers) - const [ ,videoHttp, videoMagnet, videoTorrent ] = res.body.data - await checkVideosServer1(server.url, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid) - } - }) + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + + expect(video.name).to.equal('E2E tests') + } + } + }) + + after(async function () { + await cleanupTests(servers) + }) + }) + } + + runSuite('youtube-dl') + + runSuite('yt-dlp') + + describe('Auto update', function () { + let server: PeerTubeServer - it('Should import a video that will be transcoded', async function () { - this.timeout(120000) + function quickPeerTubeImport () { + const attributes = { + targetUrl: FIXTURE_URLS.peertube_long, + channelId: server.store.channel.id, + privacy: VideoPrivacy.PUBLIC + } - const attributes = { - name: 'transcoded video', - magnetUri: getMagnetURI(), - channelId: channelIdServer2, - privacy: VideoPrivacy.PUBLIC + return server.imports.importVideo({ attributes }) } - const res = await importVideo(servers[1].url, servers[1].accessToken, attributes) - const videoUUID = res.body.video.uuid - await waitJobs(servers) + async function testBinaryUpdate (releaseUrl: string, releaseName: string) { + await remove(join(server.servers.buildDirectory('bin'), releaseName)) + + await server.kill() + await server.run({ + import: { + videos: { + http: { + youtube_dl_release: { + url: releaseUrl, + name: releaseName + } + } + } + } + }) + + await quickPeerTubeImport() - for (const server of servers) { - const res = await getVideo(server.url, videoUUID) - const video: VideoDetails = res.body + const base = server.servers.buildDirectory('bin') + const content = await readdir(base) + const binaryPath = join(base, releaseName) - expect(video.name).to.equal('transcoded video') - expect(video.files).to.have.lengthOf(4) + expect(await pathExists(binaryPath), `${binaryPath} does not exist in ${base} (${content.join(', ')})`).to.be.true } - }) - after(async function () { - killallServers(servers) + before(async function () { + this.timeout(30_000) + + // Run servers + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + }) + + it('Should update youtube-dl from github URL', async function () { + this.timeout(120_000) + + await testBinaryUpdate('https://api.github.com/repos/ytdl-org/youtube-dl/releases', 'youtube-dl') + }) + + it('Should update youtube-dl from raw URL', async function () { + this.timeout(120_000) + + await testBinaryUpdate('https://yt-dl.org/downloads/latest/youtube-dl', 'youtube-dl') + }) + + it('Should update youtube-dl from youtube-dl fork', async function () { + this.timeout(120_000) + + await testBinaryUpdate('https://api.github.com/repos/yt-dlp/yt-dlp/releases', 'yt-dlp') + }) + + after(async function () { + await cleanupTests([ server ]) + }) }) })