X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=shared%2Fextra-utils%2Fvideos%2Fvideos.ts;h=469ea4d638cf9f3014114ade4f0b545b69674565;hb=bbb3be686ad960c5b1f34a7771dec09eb1783455;hp=a0143b0efef50ac8f227a78372b7ff3af9e084c4;hpb=1fd61899eaea245a5844e33e21f04b2562f16e5e;p=github%2FChocobozzz%2FPeerTube.git diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts index a0143b0ef..469ea4d63 100644 --- a/shared/extra-utils/videos/videos.ts +++ b/shared/extra-utils/videos/videos.ts @@ -1,12 +1,14 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */ import { expect } from 'chai' -import { pathExists, readdir, readFile } from 'fs-extra' +import { createReadStream, pathExists, readdir, readFile, stat } from 'fs-extra' +import got, { Response as GotResponse } from 'got/dist/source' import * as parseTorrent from 'parse-torrent' -import { extname, join } from 'path' +import { join } from 'path' import * as request from 'supertest' -import { v4 as uuidv4 } from 'uuid' import validator from 'validator' +import { getLowercaseExtension } from '@server/helpers/core-utils' +import { buildUUID } from '@server/helpers/uuid' import { HttpStatusCode } from '@shared/core-utils' import { VideosCommonQuery } from '@shared/models' import { loadLanguages, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../server/initializers/constants' @@ -42,6 +44,7 @@ type VideoAttributes = { channelId?: number privacy?: VideoPrivacy fixture?: string + support?: string thumbnailfile?: string previewfile?: string scheduleUpdate?: { @@ -364,8 +367,13 @@ async function checkVideoFilesWereRemoved ( } } -async function uploadVideo (url: string, accessToken: string, videoAttributesArg: VideoAttributes, specialStatus = HttpStatusCode.OK_200) { - const path = '/api/v1/videos/upload' +async function uploadVideo ( + url: string, + accessToken: string, + videoAttributesArg: VideoAttributes, + specialStatus = HttpStatusCode.OK_200, + mode: 'legacy' | 'resumable' = 'legacy' +) { let defaultChannelId = '1' try { @@ -391,74 +399,170 @@ async function uploadVideo (url: string, accessToken: string, videoAttributesArg fixture: 'video_short.webm' }, videoAttributesArg) + const res = mode === 'legacy' + ? await buildLegacyUpload(url, accessToken, attributes, specialStatus) + : await buildResumeUpload(url, accessToken, attributes, specialStatus) + + // Wait torrent generation + if (specialStatus === HttpStatusCode.OK_200) { + let video: VideoDetails + do { + const resVideo = await getVideoWithToken(url, accessToken, res.body.video.uuid) + video = resVideo.body + + await wait(50) + } while (!video.files[0].torrentUrl) + } + + return res +} + +function checkUploadVideoParam ( + url: string, + token: string, + attributes: Partial, + specialStatus = HttpStatusCode.OK_200, + mode: 'legacy' | 'resumable' = 'legacy' +) { + return mode === 'legacy' + ? buildLegacyUpload(url, token, attributes, specialStatus) + : buildResumeUpload(url, token, attributes, specialStatus) +} + +async function buildLegacyUpload (url: string, token: string, attributes: VideoAttributes, specialStatus = HttpStatusCode.OK_200) { + const path = '/api/v1/videos/upload' const req = request(url) .post(path) .set('Accept', 'application/json') - .set('Authorization', 'Bearer ' + accessToken) - .field('name', attributes.name) - .field('nsfw', JSON.stringify(attributes.nsfw)) - .field('commentsEnabled', JSON.stringify(attributes.commentsEnabled)) - .field('downloadEnabled', JSON.stringify(attributes.downloadEnabled)) - .field('waitTranscoding', JSON.stringify(attributes.waitTranscoding)) - .field('privacy', attributes.privacy.toString()) - .field('channelId', attributes.channelId) - - if (attributes.support !== undefined) { - req.field('support', attributes.support) - } + .set('Authorization', 'Bearer ' + token) - if (attributes.description !== undefined) { - req.field('description', attributes.description) - } - if (attributes.language !== undefined) { - req.field('language', attributes.language.toString()) - } - if (attributes.category !== undefined) { - req.field('category', attributes.category.toString()) - } - if (attributes.licence !== undefined) { - req.field('licence', attributes.licence.toString()) - } + buildUploadReq(req, attributes) - const tags = attributes.tags || [] - for (let i = 0; i < tags.length; i++) { - req.field('tags[' + i + ']', attributes.tags[i]) + if (attributes.fixture !== undefined) { + req.attach('videofile', buildAbsoluteFixturePath(attributes.fixture)) } - if (attributes.thumbnailfile !== undefined) { - req.attach('thumbnailfile', buildAbsoluteFixturePath(attributes.thumbnailfile)) - } - if (attributes.previewfile !== undefined) { - req.attach('previewfile', buildAbsoluteFixturePath(attributes.previewfile)) - } + return req.expect(specialStatus) +} - if (attributes.scheduleUpdate) { - req.field('scheduleUpdate[updateAt]', attributes.scheduleUpdate.updateAt) +async function buildResumeUpload (url: string, token: string, attributes: VideoAttributes, specialStatus = HttpStatusCode.OK_200) { + let size = 0 + let videoFilePath: string + let mimetype = 'video/mp4' - if (attributes.scheduleUpdate.privacy) { - req.field('scheduleUpdate[privacy]', attributes.scheduleUpdate.privacy) + if (attributes.fixture) { + videoFilePath = buildAbsoluteFixturePath(attributes.fixture) + size = (await stat(videoFilePath)).size + + if (videoFilePath.endsWith('.mkv')) { + mimetype = 'video/x-matroska' + } else if (videoFilePath.endsWith('.webm')) { + mimetype = 'video/webm' } } - if (attributes.originallyPublishedAt !== undefined) { - req.field('originallyPublishedAt', attributes.originallyPublishedAt) + const initializeSessionRes = await prepareResumableUpload({ url, token, attributes, size, mimetype }) + const initStatus = initializeSessionRes.status + + if (videoFilePath && initStatus === HttpStatusCode.CREATED_201) { + const locationHeader = initializeSessionRes.header['location'] + expect(locationHeader).to.not.be.undefined + + const pathUploadId = locationHeader.split('?')[1] + + return sendResumableChunks({ url, token, pathUploadId, videoFilePath, size, specialStatus }) } - const res = await req.attach('videofile', buildAbsoluteFixturePath(attributes.fixture)) - .expect(specialStatus) + const expectedInitStatus = specialStatus === HttpStatusCode.OK_200 + ? HttpStatusCode.CREATED_201 + : specialStatus - // Wait torrent generation - if (specialStatus === HttpStatusCode.OK_200) { - let video: VideoDetails - do { - const resVideo = await getVideoWithToken(url, accessToken, res.body.video.uuid) - video = resVideo.body + expect(initStatus).to.equal(expectedInitStatus) - await wait(50) - } while (!video.files[0].torrentUrl) + return initializeSessionRes +} + +async function prepareResumableUpload (options: { + url: string + token: string + attributes: VideoAttributes + size: number + mimetype: string +}) { + const { url, token, attributes, size, mimetype } = options + + const path = '/api/v1/videos/upload-resumable' + + const req = request(url) + .post(path) + .set('Authorization', 'Bearer ' + token) + .set('X-Upload-Content-Type', mimetype) + .set('X-Upload-Content-Length', size.toString()) + + buildUploadReq(req, attributes) + + if (attributes.fixture) { + req.field('filename', attributes.fixture) } - return res + return req +} + +function sendResumableChunks (options: { + url: string + token: string + pathUploadId: string + videoFilePath: string + size: number + specialStatus?: HttpStatusCode + contentLength?: number + contentRangeBuilder?: (start: number, chunk: any) => string +}) { + const { url, token, pathUploadId, videoFilePath, size, specialStatus, contentLength, contentRangeBuilder } = options + + const expectedStatus = specialStatus || HttpStatusCode.OK_200 + + const path = '/api/v1/videos/upload-resumable' + let start = 0 + + const readable = createReadStream(videoFilePath, { highWaterMark: 8 * 1024 }) + return new Promise((resolve, reject) => { + readable.on('data', async function onData (chunk) { + readable.pause() + + const headers = { + 'Authorization': 'Bearer ' + token, + 'Content-Type': 'application/octet-stream', + 'Content-Range': contentRangeBuilder + ? contentRangeBuilder(start, chunk) + : `bytes ${start}-${start + chunk.length - 1}/${size}`, + 'Content-Length': contentLength ? contentLength + '' : chunk.length + '' + } + + const res = await got({ + url, + method: 'put', + headers, + path: path + '?' + pathUploadId, + body: chunk, + responseType: 'json', + throwHttpErrors: false + }) + + start += chunk.length + + if (res.statusCode === expectedStatus) { + return resolve(res) + } + + if (res.statusCode !== HttpStatusCode.PERMANENT_REDIRECT_308) { + readable.off('data', onData) + return reject(new Error('Incorrect transient behaviour sending intermediary chunks')) + } + + readable.resume() + }) + }) } function updateVideo ( @@ -635,7 +739,7 @@ async function completeVideoCheck ( const file = videoDetails.files.find(f => f.resolution.id === attributeFile.resolution) expect(file).not.to.be.undefined - let extension = extname(attributes.fixture) + let extension = getLowercaseExtension(attributes.fixture) // Transcoding enabled: extension will always be .mp4 if (attributes.files.length > 1) extension = '.mp4' @@ -671,9 +775,11 @@ async function completeVideoCheck ( expect(torrent.files[0].path).to.exist.and.to.not.equal('') } + expect(videoDetails.thumbnailPath).to.exist await testImage(url, attributes.thumbnailfile || attributes.fixture, videoDetails.thumbnailPath) if (attributes.previewfile) { + expect(videoDetails.previewPath).to.exist await testImage(url, attributes.previewfile, videoDetails.previewPath) } } @@ -700,7 +806,7 @@ async function uploadVideoAndGetId (options: { const res = await uploadVideo(options.server.url, options.token || options.server.accessToken, videoAttrs) - return { id: res.body.video.id, uuid: res.body.video.uuid } + return res.body.video as { id: number, uuid: string, shortUUID: string } } async function getLocalIdByUUID (url: string, uuid: string) { @@ -721,7 +827,7 @@ async function uploadRandomVideoOnServers (servers: ServerInfo[], serverNumber: async function uploadRandomVideo (server: ServerInfo, wait = true, additionalParams: any = {}) { const prefixName = additionalParams.prefixName || '' - const name = prefixName + uuidv4() + const name = prefixName + buildUUID() const data = Object.assign({ name }, additionalParams) const res = await uploadVideo(server.url, server.accessToken, data) @@ -749,11 +855,13 @@ export { getVideoWithToken, getVideosList, removeAllVideos, + checkUploadVideoParam, getVideosListPagination, getVideosListSort, removeVideo, getVideosListWithToken, uploadVideo, + sendResumableChunks, getVideosWithFilters, uploadRandomVideoOnServers, updateVideo, @@ -767,5 +875,50 @@ export { getMyVideosWithFilter, uploadVideoAndGetId, getLocalIdByUUID, - getVideoIdFromUUID + getVideoIdFromUUID, + prepareResumableUpload +} + +// --------------------------------------------------------------------------- + +function buildUploadReq (req: request.Test, attributes: VideoAttributes) { + + for (const key of [ 'name', 'support', 'channelId', 'description', 'originallyPublishedAt' ]) { + if (attributes[key] !== undefined) { + req.field(key, attributes[key]) + } + } + + for (const key of [ 'nsfw', 'commentsEnabled', 'downloadEnabled', 'waitTranscoding' ]) { + if (attributes[key] !== undefined) { + req.field(key, JSON.stringify(attributes[key])) + } + } + + for (const key of [ 'language', 'privacy', 'category', 'licence' ]) { + if (attributes[key] !== undefined) { + req.field(key, attributes[key].toString()) + } + } + + const tags = attributes.tags || [] + for (let i = 0; i < tags.length; i++) { + req.field('tags[' + i + ']', attributes.tags[i]) + } + + for (const key of [ 'thumbnailfile', 'previewfile' ]) { + if (attributes[key] !== undefined) { + req.attach(key, buildAbsoluteFixturePath(attributes[key])) + } + } + + if (attributes.scheduleUpdate) { + if (attributes.scheduleUpdate.updateAt) { + req.field('scheduleUpdate[updateAt]', attributes.scheduleUpdate.updateAt) + } + + if (attributes.scheduleUpdate.privacy) { + req.field('scheduleUpdate[privacy]', attributes.scheduleUpdate.privacy) + } + } }