From 60f1f61579947caab1b1d23646cd4e82691b431c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 6 May 2021 16:39:17 +0200 Subject: Fix ffmpeg version checker --- server/helpers/ffmpeg-utils.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'server/helpers') diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index 75297df8f..f79b70469 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts @@ -679,10 +679,16 @@ function getFFmpegVersion () { return execPromise(`${ffmpegPath} -version`) .then(stdout => { - const parsed = stdout.match(/ffmpeg version .?(\d+\.\d+\.\d+)/) + const parsed = stdout.match(/ffmpeg version .?(\d+\.\d+(\.\d+)?)/) if (!parsed || !parsed[1]) return rej(new Error(`Could not find ffmpeg version in ${stdout}`)) - return res(parsed[1]) + // Fix ffmpeg version that does not include patch version (4.4 for example) + let version = parsed[1] + if (version.match(/^\d+\.\d+/)) { + version += '.0' + } + + return res(version) }) .catch(err => rej(err)) }) -- cgit v1.2.3 From a66c2e3252d6cca8958959966f42494ded564023 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 7 May 2021 08:59:59 +0200 Subject: Fix remote actor creation date --- .../helpers/custom-validators/activitypub/actor.ts | 39 ++++++++++------------ 1 file changed, 18 insertions(+), 21 deletions(-) (limited to 'server/helpers') diff --git a/server/helpers/custom-validators/activitypub/actor.ts b/server/helpers/custom-validators/activitypub/actor.ts index 877345157..675a7b663 100644 --- a/server/helpers/custom-validators/activitypub/actor.ts +++ b/server/helpers/custom-validators/activitypub/actor.ts @@ -1,6 +1,6 @@ import validator from 'validator' import { CONSTRAINTS_FIELDS } from '../../../initializers/constants' -import { exists, isArray } from '../misc' +import { exists, isArray, isDateValid } from '../misc' import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc' import { isHostValid } from '../servers' import { peertubeTruncate } from '@server/helpers/core-utils' @@ -47,7 +47,21 @@ function isActorPrivateKeyValid (privateKey: string) { validator.isLength(privateKey, CONSTRAINTS_FIELDS.ACTORS.PRIVATE_KEY) } -function isActorObjectValid (actor: any) { +function isActorFollowingCountValid (value: string) { + return exists(value) && validator.isInt('' + value, { min: 0 }) +} + +function isActorFollowersCountValid (value: string) { + return exists(value) && validator.isInt('' + value, { min: 0 }) +} + +function isActorDeleteActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Delete') +} + +function sanitizeAndCheckActorObject (actor: any) { + normalizeActor(actor) + return exists(actor) && isActivityPubUrlValid(actor.id) && isActorTypeValid(actor.type) && @@ -68,24 +82,6 @@ function isActorObjectValid (actor: any) { (actor.type !== 'Group' || actor.attributedTo.length !== 0) } -function isActorFollowingCountValid (value: string) { - return exists(value) && validator.isInt('' + value, { min: 0 }) -} - -function isActorFollowersCountValid (value: string) { - return exists(value) && validator.isInt('' + value, { min: 0 }) -} - -function isActorDeleteActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Delete') -} - -function sanitizeAndCheckActorObject (object: any) { - normalizeActor(object) - - return isActorObjectValid(object) -} - function normalizeActor (actor: any) { if (!actor) return @@ -95,6 +91,8 @@ function normalizeActor (actor: any) { actor.url = actor.url.href || actor.url.url } + if (!isDateValid(actor.published)) actor.published = undefined + if (actor.summary && typeof actor.summary === 'string') { actor.summary = peertubeTruncate(actor.summary, { length: CONSTRAINTS_FIELDS.USERS.DESCRIPTION.max }) @@ -135,7 +133,6 @@ export { isActorPublicKeyValid, isActorPreferredUsernameValid, isActorPrivateKeyValid, - isActorObjectValid, isActorFollowingCountValid, isActorFollowersCountValid, isActorDeleteActivityValid, -- cgit v1.2.3 From 1ff9f1cda3f8283241776cd8344e6671742b25f7 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 7 May 2021 14:23:43 +0200 Subject: Fix ffmpeg version parsing --- server/helpers/ffmpeg-utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/helpers') diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index f79b70469..25d9d4951 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts @@ -684,7 +684,7 @@ function getFFmpegVersion () { // Fix ffmpeg version that does not include patch version (4.4 for example) let version = parsed[1] - if (version.match(/^\d+\.\d+/)) { + if (version.match(/^\d+\.\d+$/)) { version += '.0' } -- cgit v1.2.3 From e024fd6a7494b37251da1d59470324305cdb4129 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 7 May 2021 17:14:39 +0200 Subject: Update channel updatedAt when uploading a video --- server/helpers/database-utils.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'server/helpers') diff --git a/server/helpers/database-utils.ts b/server/helpers/database-utils.ts index 2b916efc2..f9cb33aca 100644 --- a/server/helpers/database-utils.ts +++ b/server/helpers/database-utils.ts @@ -1,8 +1,9 @@ import * as retry from 'async/retry' import * as Bluebird from 'bluebird' +import { QueryTypes, Transaction } from 'sequelize' import { Model } from 'sequelize-typescript' +import { sequelizeTypescript } from '@server/initializers/database' import { logger } from './logger' -import { Transaction } from 'sequelize' function retryTransactionWrapper ( functionToRetry: (arg1: A, arg2: B, arg3: C, arg4: D) => Promise | Bluebird, @@ -96,6 +97,18 @@ function deleteNonExistingModels f.destroy({ transaction: t })) } +// Sequelize always skip the update if we only update updatedAt field +function setAsUpdated (table: string, id: number, transaction?: Transaction) { + return sequelizeTypescript.query( + `UPDATE "${table}" SET "updatedAt" = :updatedAt WHERE id = :id`, + { + replacements: { table, id, updatedAt: new Date() }, + type: QueryTypes.UPDATE, + transaction + } + ) +} + // --------------------------------------------------------------------------- export { @@ -104,5 +117,6 @@ export { transactionRetryer, updateInstanceWithAnother, afterCommitIfTransaction, - deleteNonExistingModels + deleteNonExistingModels, + setAsUpdated } -- cgit v1.2.3 From f6d6e7f861189a4446f406efb775a29688764b48 Mon Sep 17 00:00:00 2001 From: kontrollanten <6680299+kontrollanten@users.noreply.github.com> Date: Mon, 10 May 2021 11:13:41 +0200 Subject: Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent Co-authored-by: Rigel Kent Co-authored-by: Chocobozzz --- server/helpers/custom-validators/misc.ts | 5 +++-- server/helpers/custom-validators/videos.ts | 9 +++++---- server/helpers/express-utils.ts | 8 ++++---- server/helpers/upload.ts | 21 +++++++++++++++++++++ server/helpers/utils.ts | 4 ++-- 5 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 server/helpers/upload.ts (limited to 'server/helpers') diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts index effdd98cb..fd3b45804 100644 --- a/server/helpers/custom-validators/misc.ts +++ b/server/helpers/custom-validators/misc.ts @@ -1,6 +1,7 @@ import 'multer' -import validator from 'validator' +import { UploadFilesForCheck } from 'express' import { sep } from 'path' +import validator from 'validator' function exists (value: any) { return value !== undefined && value !== null @@ -108,7 +109,7 @@ function isFileFieldValid ( } function isFileMimeTypeValid ( - files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[], + files: UploadFilesForCheck, mimeTypeRegex: string, field: string, optional = false diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts index 87966798f..b33e088eb 100644 --- a/server/helpers/custom-validators/videos.ts +++ b/server/helpers/custom-validators/videos.ts @@ -1,4 +1,6 @@ +import { UploadFilesForCheck } from 'express' import { values } from 'lodash' +import * as magnetUtil from 'magnet-uri' import validator from 'validator' import { VideoFilter, VideoPrivacy, VideoRateType } from '../../../shared' import { @@ -6,13 +8,12 @@ import { MIMETYPES, VIDEO_CATEGORIES, VIDEO_LICENCES, + VIDEO_LIVE, VIDEO_PRIVACIES, VIDEO_RATE_TYPES, - VIDEO_STATES, - VIDEO_LIVE + VIDEO_STATES } from '../../initializers/constants' import { exists, isArray, isDateValid, isFileMimeTypeValid, isFileValid } from './misc' -import * as magnetUtil from 'magnet-uri' const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS @@ -81,7 +82,7 @@ function isVideoFileExtnameValid (value: string) { return exists(value) && (value === VIDEO_LIVE.EXTENSION || MIMETYPES.VIDEO.EXT_MIMETYPE[value] !== undefined) } -function isVideoFileMimeTypeValid (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) { +function isVideoFileMimeTypeValid (files: UploadFilesForCheck) { return isFileMimeTypeValid(files, MIMETYPES.VIDEO.MIMETYPES_REGEX, 'videofile') } diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts index c0d3f8f32..ede22a3cc 100644 --- a/server/helpers/express-utils.ts +++ b/server/helpers/express-utils.ts @@ -2,7 +2,7 @@ import * as express from 'express' import * as multer from 'multer' import { REMOTE_SCHEME } from '../initializers/constants' import { logger } from './logger' -import { deleteFileAsync, generateRandomString } from './utils' +import { deleteFileAndCatch, generateRandomString } from './utils' import { extname } from 'path' import { isArray } from './custom-validators/misc' import { CONFIG } from '../initializers/config' @@ -36,15 +36,15 @@ function cleanUpReqFiles (req: { files: { [fieldname: string]: Express.Multer.Fi if (!files) return if (isArray(files)) { - (files as Express.Multer.File[]).forEach(f => deleteFileAsync(f.path)) + (files as Express.Multer.File[]).forEach(f => deleteFileAndCatch(f.path)) return } for (const key of Object.keys(files)) { const file = files[key] - if (isArray(file)) file.forEach(f => deleteFileAsync(f.path)) - else deleteFileAsync(file.path) + if (isArray(file)) file.forEach(f => deleteFileAndCatch(f.path)) + else deleteFileAndCatch(file.path) } } diff --git a/server/helpers/upload.ts b/server/helpers/upload.ts new file mode 100644 index 000000000..030a6b7d5 --- /dev/null +++ b/server/helpers/upload.ts @@ -0,0 +1,21 @@ +import { METAFILE_EXTNAME } from '@uploadx/core' +import { remove } from 'fs-extra' +import { join } from 'path' +import { RESUMABLE_UPLOAD_DIRECTORY } from '../initializers/constants' + +function getResumableUploadPath (filename?: string) { + if (filename) return join(RESUMABLE_UPLOAD_DIRECTORY, filename) + + return RESUMABLE_UPLOAD_DIRECTORY +} + +function deleteResumableUploadMetaFile (filepath: string) { + return remove(filepath + METAFILE_EXTNAME) +} + +// --------------------------------------------------------------------------- + +export { + getResumableUploadPath, + deleteResumableUploadMetaFile +} diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index 0545e8996..6c95a43b6 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts @@ -6,7 +6,7 @@ import { CONFIG } from '../initializers/config' import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils' import { logger } from './logger' -function deleteFileAsync (path: string) { +function deleteFileAndCatch (path: string) { remove(path) .catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err })) } @@ -83,7 +83,7 @@ function getUUIDFromFilename (filename: string) { // --------------------------------------------------------------------------- export { - deleteFileAsync, + deleteFileAndCatch, generateRandomString, getFormattedObjects, getSecureTorrentName, -- cgit v1.2.3 From 1bcb03a100d172903b877d6a0e4ed11d63b14f3d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 11 May 2021 10:54:05 +0200 Subject: Use a class for youtube-dl --- server/helpers/youtube-dl.ts | 553 +++++++++++++++++++++---------------------- 1 file changed, 275 insertions(+), 278 deletions(-) (limited to 'server/helpers') diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts index fac3da6ba..d003ea3cf 100644 --- a/server/helpers/youtube-dl.ts +++ b/server/helpers/youtube-dl.ts @@ -6,7 +6,6 @@ import { CONFIG } from '@server/initializers/config' import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' import { VideoResolution } from '../../shared/models/videos' import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants' -import { getEnabledResolutions } from '../lib/video-transcoding' import { peertubeTruncate, pipelinePromise, root } from './core-utils' import { isVideoFileExtnameValid } from './custom-validators/videos' import { logger } from './logger' @@ -35,361 +34,359 @@ const processOptions = { maxBuffer: 1024 * 1024 * 10 // 10MB } -function getYoutubeDLInfo (url: string, opts?: string[]): Promise { - return new Promise((res, rej) => { - let args = opts || [ '-j', '--flat-playlist' ] +class YoutubeDL { - if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) { - args.push('--force-ipv4') - } + constructor (private readonly url: string = '', private readonly enabledResolutions: number[] = []) { - args = wrapWithProxyOptions(args) - args = [ '-f', getYoutubeDLVideoFormat() ].concat(args) + } - safeGetYoutubeDL() - .then(youtubeDL => { - youtubeDL.getInfo(url, args, processOptions, (err, info) => { - if (err) return rej(err) - if (info.is_live === true) return rej(new Error('Cannot download a live streaming.')) + getYoutubeDLInfo (opts?: string[]): Promise { + return new Promise((res, rej) => { + let args = opts || [ '-j', '--flat-playlist' ] - const obj = buildVideoInfo(normalizeObject(info)) - if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video' + if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) { + args.push('--force-ipv4') + } - return res(obj) - }) - }) - .catch(err => rej(err)) - }) -} + args = this.wrapWithProxyOptions(args) + args = [ '-f', this.getYoutubeDLVideoFormat() ].concat(args) -function getYoutubeDLSubs (url: string, opts?: object): Promise { - return new Promise((res, rej) => { - const cwd = CONFIG.STORAGE.TMP_DIR - const options = opts || { all: true, format: 'vtt', cwd } - - safeGetYoutubeDL() - .then(youtubeDL => { - youtubeDL.getSubs(url, options, (err, files) => { - if (err) return rej(err) - if (!files) return [] - - logger.debug('Get subtitles from youtube dl.', { url, files }) - - const subtitles = files.reduce((acc, filename) => { - const matched = filename.match(/\.([a-z]{2})(-[a-z]+)?\.(vtt|ttml)/i) - if (!matched || !matched[1]) return acc - - return [ - ...acc, - { - language: matched[1], - path: join(cwd, filename), - filename - } - ] - }, []) + YoutubeDL.safeGetYoutubeDL() + .then(youtubeDL => { + youtubeDL.getInfo(this.url, args, processOptions, (err, info) => { + if (err) return rej(err) + if (info.is_live === true) return rej(new Error('Cannot download a live streaming.')) - return res(subtitles) + const obj = this.buildVideoInfo(this.normalizeObject(info)) + if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video' + + return res(obj) + }) }) - }) - .catch(err => rej(err)) - }) -} + .catch(err => rej(err)) + }) + } -function getYoutubeDLVideoFormat () { - /** - * list of format selectors in order or preference - * see https://github.com/ytdl-org/youtube-dl#format-selection - * - * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope - * of being able to do a "quick-transcode" - * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9) - * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback - * - * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499 - **/ - const enabledResolutions = getEnabledResolutions('vod') - const resolution = enabledResolutions.length === 0 - ? VideoResolution.H_720P - : Math.max(...enabledResolutions) - - return [ - `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1 - `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2 - `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]`, // case #3 - `bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio`, - 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats - 'best' // Ultimate fallback - ].join('/') -} + getYoutubeDLSubs (opts?: object): Promise { + return new Promise((res, rej) => { + const cwd = CONFIG.STORAGE.TMP_DIR + const options = opts || { all: true, format: 'vtt', cwd } + + YoutubeDL.safeGetYoutubeDL() + .then(youtubeDL => { + youtubeDL.getSubs(this.url, options, (err, files) => { + if (err) return rej(err) + if (!files) return [] + + logger.debug('Get subtitles from youtube dl.', { url: this.url, files }) + + const subtitles = files.reduce((acc, filename) => { + const matched = filename.match(/\.([a-z]{2})(-[a-z]+)?\.(vtt|ttml)/i) + if (!matched || !matched[1]) return acc + + return [ + ...acc, + { + language: matched[1], + path: join(cwd, filename), + filename + } + ] + }, []) + + return res(subtitles) + }) + }) + .catch(err => rej(err)) + }) + } -function downloadYoutubeDLVideo (url: string, fileExt: string, timeout: number) { - // Leave empty the extension, youtube-dl will add it - const pathWithoutExtension = generateVideoImportTmpPath(url, '') + getYoutubeDLVideoFormat () { + /** + * list of format selectors in order or preference + * see https://github.com/ytdl-org/youtube-dl#format-selection + * + * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope + * of being able to do a "quick-transcode" + * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9) + * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback + * + * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499 + **/ + const resolution = this.enabledResolutions.length === 0 + ? VideoResolution.H_720P + : Math.max(...this.enabledResolutions) + + return [ + `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1 + `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2 + `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]`, // case #3 + `bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio`, + 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats + 'best' // Ultimate fallback + ].join('/') + } - let timer + downloadYoutubeDLVideo (fileExt: string, timeout: number) { + // Leave empty the extension, youtube-dl will add it + const pathWithoutExtension = generateVideoImportTmpPath(this.url, '') - logger.info('Importing youtubeDL video %s to %s', url, pathWithoutExtension) + let timer - let options = [ '-f', getYoutubeDLVideoFormat(), '-o', pathWithoutExtension ] - options = wrapWithProxyOptions(options) + logger.info('Importing youtubeDL video %s to %s', this.url, pathWithoutExtension) - if (process.env.FFMPEG_PATH) { - options = options.concat([ '--ffmpeg-location', process.env.FFMPEG_PATH ]) - } + let options = [ '-f', this.getYoutubeDLVideoFormat(), '-o', pathWithoutExtension ] + options = this.wrapWithProxyOptions(options) - logger.debug('YoutubeDL options for %s.', url, { options }) + if (process.env.FFMPEG_PATH) { + options = options.concat([ '--ffmpeg-location', process.env.FFMPEG_PATH ]) + } - return new Promise((res, rej) => { - safeGetYoutubeDL() - .then(youtubeDL => { - youtubeDL.exec(url, options, processOptions, async err => { - clearTimeout(timer) + logger.debug('YoutubeDL options for %s.', this.url, { options }) - try { - // If youtube-dl did not guess an extension for our file, just use .mp4 as default - if (await pathExists(pathWithoutExtension)) { - await move(pathWithoutExtension, pathWithoutExtension + '.mp4') - } + return new Promise((res, rej) => { + YoutubeDL.safeGetYoutubeDL() + .then(youtubeDL => { + youtubeDL.exec(this.url, options, processOptions, async err => { + clearTimeout(timer) + + try { + // If youtube-dl did not guess an extension for our file, just use .mp4 as default + if (await pathExists(pathWithoutExtension)) { + await move(pathWithoutExtension, pathWithoutExtension + '.mp4') + } - const path = await guessVideoPathWithExtension(pathWithoutExtension, fileExt) + const path = await this.guessVideoPathWithExtension(pathWithoutExtension, fileExt) - if (err) { - remove(path) - .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err })) + if (err) { + remove(path) + .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err })) + return rej(err) + } + + return res(path) + } catch (err) { return rej(err) } - - return res(path) - } catch (err) { - return rej(err) - } + }) + + timer = setTimeout(() => { + const err = new Error('YoutubeDL download timeout.') + + this.guessVideoPathWithExtension(pathWithoutExtension, fileExt) + .then(path => remove(path)) + .finally(() => rej(err)) + .catch(err => { + logger.error('Cannot remove file in youtubeDL timeout.', { err }) + return rej(err) + }) + }, timeout) }) + .catch(err => rej(err)) + }) + } - timer = setTimeout(() => { - const err = new Error('YoutubeDL download timeout.') + buildOriginallyPublishedAt (obj: any) { + let originallyPublishedAt: Date = null - guessVideoPathWithExtension(pathWithoutExtension, fileExt) - .then(path => remove(path)) - .finally(() => rej(err)) - .catch(err => { - logger.error('Cannot remove file in youtubeDL timeout.', { err }) - return rej(err) - }) - }, timeout) - }) - .catch(err => rej(err)) - }) -} - -// Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js -// We rewrote it to avoid sync calls -async function updateYoutubeDLBinary () { - logger.info('Updating youtubeDL binary.') + const uploadDateMatcher = /^(\d{4})(\d{2})(\d{2})$/.exec(obj.upload_date) + if (uploadDateMatcher) { + originallyPublishedAt = new Date() + originallyPublishedAt.setHours(0, 0, 0, 0) - const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin') - const bin = join(binDirectory, 'youtube-dl') - const detailsPath = join(binDirectory, 'details') - const url = process.env.YOUTUBE_DL_DOWNLOAD_HOST || 'https://yt-dl.org/downloads/latest/youtube-dl' + const year = parseInt(uploadDateMatcher[1], 10) + // Month starts from 0 + const month = parseInt(uploadDateMatcher[2], 10) - 1 + const day = parseInt(uploadDateMatcher[3], 10) - await ensureDir(binDirectory) + originallyPublishedAt.setFullYear(year, month, day) + } - try { - const result = await got(url, { followRedirect: false }) + return originallyPublishedAt + } - if (result.statusCode !== HttpStatusCode.FOUND_302) { - logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode) - return + private async guessVideoPathWithExtension (tmpPath: string, sourceExt: string) { + if (!isVideoFileExtnameValid(sourceExt)) { + throw new Error('Invalid video extension ' + sourceExt) } - const newUrl = result.headers.location - const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(newUrl)[1] + const extensions = [ sourceExt, '.mp4', '.mkv', '.webm' ] - const downloadFileStream = got.stream(newUrl) - const writeStream = createWriteStream(bin, { mode: 493 }) + for (const extension of extensions) { + const path = tmpPath + extension - await pipelinePromise( - downloadFileStream, - writeStream - ) - - const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' }) - await writeFile(detailsPath, details, { encoding: 'utf8' }) + if (await pathExists(path)) return path + } - logger.info('youtube-dl updated to version %s.', newVersion) - } catch (err) { - logger.error('Cannot update youtube-dl.', { err }) + throw new Error('Cannot guess path of ' + tmpPath) } -} -async function safeGetYoutubeDL () { - let youtubeDL + private normalizeObject (obj: any) { + const newObj: any = {} - try { - youtubeDL = require('youtube-dl') - } catch (e) { - // Download binary - await updateYoutubeDLBinary() - youtubeDL = require('youtube-dl') - } + for (const key of Object.keys(obj)) { + // Deprecated key + if (key === 'resolution') continue - return youtubeDL -} + const value = obj[key] -function buildOriginallyPublishedAt (obj: any) { - let originallyPublishedAt: Date = null + if (typeof value === 'string') { + newObj[key] = value.normalize() + } else { + newObj[key] = value + } + } - const uploadDateMatcher = /^(\d{4})(\d{2})(\d{2})$/.exec(obj.upload_date) - if (uploadDateMatcher) { - originallyPublishedAt = new Date() - originallyPublishedAt.setHours(0, 0, 0, 0) + return newObj + } - const year = parseInt(uploadDateMatcher[1], 10) - // Month starts from 0 - const month = parseInt(uploadDateMatcher[2], 10) - 1 - const day = parseInt(uploadDateMatcher[3], 10) + private buildVideoInfo (obj: any): YoutubeDLInfo { + return { + name: this.titleTruncation(obj.title), + description: this.descriptionTruncation(obj.description), + category: this.getCategory(obj.categories), + licence: this.getLicence(obj.license), + language: this.getLanguage(obj.language), + nsfw: this.isNSFW(obj), + tags: this.getTags(obj.tags), + thumbnailUrl: obj.thumbnail || undefined, + originallyPublishedAt: this.buildOriginallyPublishedAt(obj), + ext: obj.ext + } + } - originallyPublishedAt.setFullYear(year, month, day) + private titleTruncation (title: string) { + return peertubeTruncate(title, { + length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max, + separator: /,? +/, + omission: ' […]' + }) } - return originallyPublishedAt -} + private descriptionTruncation (description: string) { + if (!description || description.length < CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.min) return undefined -// --------------------------------------------------------------------------- + return peertubeTruncate(description, { + length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max, + separator: /,? +/, + omission: ' […]' + }) + } -export { - updateYoutubeDLBinary, - getYoutubeDLVideoFormat, - downloadYoutubeDLVideo, - getYoutubeDLSubs, - getYoutubeDLInfo, - safeGetYoutubeDL, - buildOriginallyPublishedAt -} + private isNSFW (info: any) { + return info.age_limit && info.age_limit >= 16 + } -// --------------------------------------------------------------------------- + private getTags (tags: any) { + if (Array.isArray(tags) === false) return [] -async function guessVideoPathWithExtension (tmpPath: string, sourceExt: string) { - if (!isVideoFileExtnameValid(sourceExt)) { - throw new Error('Invalid video extension ' + sourceExt) + return tags + .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min) + .map(t => t.normalize()) + .slice(0, 5) } - const extensions = [ sourceExt, '.mp4', '.mkv', '.webm' ] + private getLicence (licence: string) { + if (!licence) return undefined - for (const extension of extensions) { - const path = tmpPath + extension + if (licence.includes('Creative Commons Attribution')) return 1 - if (await pathExists(path)) return path - } + for (const key of Object.keys(VIDEO_LICENCES)) { + const peertubeLicence = VIDEO_LICENCES[key] + if (peertubeLicence.toLowerCase() === licence.toLowerCase()) return parseInt(key, 10) + } - throw new Error('Cannot guess path of ' + tmpPath) -} + return undefined + } -function normalizeObject (obj: any) { - const newObj: any = {} + private getCategory (categories: string[]) { + if (!categories) return undefined - for (const key of Object.keys(obj)) { - // Deprecated key - if (key === 'resolution') continue + const categoryString = categories[0] + if (!categoryString || typeof categoryString !== 'string') return undefined - const value = obj[key] + if (categoryString === 'News & Politics') return 11 - if (typeof value === 'string') { - newObj[key] = value.normalize() - } else { - newObj[key] = value + for (const key of Object.keys(VIDEO_CATEGORIES)) { + const category = VIDEO_CATEGORIES[key] + if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10) } - } - return newObj -} - -function buildVideoInfo (obj: any): YoutubeDLInfo { - return { - name: titleTruncation(obj.title), - description: descriptionTruncation(obj.description), - category: getCategory(obj.categories), - licence: getLicence(obj.license), - language: getLanguage(obj.language), - nsfw: isNSFW(obj), - tags: getTags(obj.tags), - thumbnailUrl: obj.thumbnail || undefined, - originallyPublishedAt: buildOriginallyPublishedAt(obj), - ext: obj.ext + return undefined } -} -function titleTruncation (title: string) { - return peertubeTruncate(title, { - length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max, - separator: /,? +/, - omission: ' […]' - }) -} + private getLanguage (language: string) { + return VIDEO_LANGUAGES[language] ? language : undefined + } -function descriptionTruncation (description: string) { - if (!description || description.length < CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.min) return undefined + private wrapWithProxyOptions (options: string[]) { + if (CONFIG.IMPORT.VIDEOS.HTTP.PROXY.ENABLED) { + logger.debug('Using proxy for YoutubeDL') - return peertubeTruncate(description, { - length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max, - separator: /,? +/, - omission: ' […]' - }) -} + return [ '--proxy', CONFIG.IMPORT.VIDEOS.HTTP.PROXY.URL ].concat(options) + } -function isNSFW (info: any) { - return info.age_limit && info.age_limit >= 16 -} + return options + } -function getTags (tags: any) { - if (Array.isArray(tags) === false) return [] + // Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js + // We rewrote it to avoid sync calls + static async updateYoutubeDLBinary () { + logger.info('Updating youtubeDL binary.') - return tags - .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min) - .map(t => t.normalize()) - .slice(0, 5) -} + const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin') + const bin = join(binDirectory, 'youtube-dl') + const detailsPath = join(binDirectory, 'details') + const url = process.env.YOUTUBE_DL_DOWNLOAD_HOST || 'https://yt-dl.org/downloads/latest/youtube-dl' -function getLicence (licence: string) { - if (!licence) return undefined + await ensureDir(binDirectory) - if (licence.includes('Creative Commons Attribution')) return 1 + try { + const result = await got(url, { followRedirect: false }) - for (const key of Object.keys(VIDEO_LICENCES)) { - const peertubeLicence = VIDEO_LICENCES[key] - if (peertubeLicence.toLowerCase() === licence.toLowerCase()) return parseInt(key, 10) - } + if (result.statusCode !== HttpStatusCode.FOUND_302) { + logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode) + return + } - return undefined -} + const newUrl = result.headers.location + const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(newUrl)[1] -function getCategory (categories: string[]) { - if (!categories) return undefined + const downloadFileStream = got.stream(newUrl) + const writeStream = createWriteStream(bin, { mode: 493 }) - const categoryString = categories[0] - if (!categoryString || typeof categoryString !== 'string') return undefined + await pipelinePromise( + downloadFileStream, + writeStream + ) - if (categoryString === 'News & Politics') return 11 + const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' }) + await writeFile(detailsPath, details, { encoding: 'utf8' }) - for (const key of Object.keys(VIDEO_CATEGORIES)) { - const category = VIDEO_CATEGORIES[key] - if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10) + logger.info('youtube-dl updated to version %s.', newVersion) + } catch (err) { + logger.error('Cannot update youtube-dl.', { err }) + } } - return undefined -} - -function getLanguage (language: string) { - return VIDEO_LANGUAGES[language] ? language : undefined -} + static async safeGetYoutubeDL () { + let youtubeDL -function wrapWithProxyOptions (options: string[]) { - if (CONFIG.IMPORT.VIDEOS.HTTP.PROXY.ENABLED) { - logger.debug('Using proxy for YoutubeDL') + try { + youtubeDL = require('youtube-dl') + } catch (e) { + // Download binary + await this.updateYoutubeDLBinary() + youtubeDL = require('youtube-dl') + } - return [ '--proxy', CONFIG.IMPORT.VIDEOS.HTTP.PROXY.URL ].concat(options) + return youtubeDL } +} + +// --------------------------------------------------------------------------- - return options +export { + YoutubeDL } -- cgit v1.2.3 From 7d9ba5c08999c6482f0bc5e0c09c6f55b7724090 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 11 May 2021 11:15:29 +0200 Subject: Cleanup models directory organization --- server/helpers/actor.ts | 2 +- server/helpers/middlewares/accounts.ts | 2 +- server/helpers/signup.ts | 2 +- server/helpers/webfinger.ts | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) (limited to 'server/helpers') diff --git a/server/helpers/actor.ts b/server/helpers/actor.ts index a60d3ed5d..5f742505b 100644 --- a/server/helpers/actor.ts +++ b/server/helpers/actor.ts @@ -1,5 +1,5 @@ -import { ActorModel } from '../models/activitypub/actor' +import { ActorModel } from '../models/actor/actor' import { MActorAccountChannelId, MActorFull } from '../types/models' type ActorFetchByUrlType = 'all' | 'association-ids' diff --git a/server/helpers/middlewares/accounts.ts b/server/helpers/middlewares/accounts.ts index 13ae6cdf4..5addd3e1a 100644 --- a/server/helpers/middlewares/accounts.ts +++ b/server/helpers/middlewares/accounts.ts @@ -1,5 +1,5 @@ import { Response } from 'express' -import { UserModel } from '@server/models/account/user' +import { UserModel } from '@server/models/user/user' import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' import { AccountModel } from '../../models/account/account' import { MAccountDefault } from '../../types/models' diff --git a/server/helpers/signup.ts b/server/helpers/signup.ts index ed872539b..8fa81e601 100644 --- a/server/helpers/signup.ts +++ b/server/helpers/signup.ts @@ -1,4 +1,4 @@ -import { UserModel } from '../models/account/user' +import { UserModel } from '../models/user/user' import * as ipaddr from 'ipaddr.js' import { CONFIG } from '../initializers/config' diff --git a/server/helpers/webfinger.ts b/server/helpers/webfinger.ts index da7e88077..33367f651 100644 --- a/server/helpers/webfinger.ts +++ b/server/helpers/webfinger.ts @@ -1,10 +1,10 @@ import * as WebFinger from 'webfinger.js' import { WebFingerData } from '../../shared' -import { ActorModel } from '../models/activitypub/actor' -import { isTestInstance } from './core-utils' -import { isActivityPubUrlValid } from './custom-validators/activitypub/misc' import { WEBSERVER } from '../initializers/constants' +import { ActorModel } from '../models/actor/actor' import { MActorFull } from '../types/models' +import { isTestInstance } from './core-utils' +import { isActivityPubUrlValid } from './custom-validators/activitypub/misc' const webfinger = new WebFinger({ webfist_fallback: false, -- cgit v1.2.3 From 2b02c520e66ea452687cab39401b371711caa9ed Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 11 May 2021 11:27:40 +0200 Subject: Cleanup shared models --- server/helpers/audit-logger.ts | 2 +- server/helpers/ffprobe-utils.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'server/helpers') diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts index 6aae5e821..884bd187d 100644 --- a/server/helpers/audit-logger.ts +++ b/server/helpers/audit-logger.ts @@ -7,7 +7,7 @@ import * as winston from 'winston' import { AUDIT_LOG_FILENAME } from '@server/initializers/constants' import { AdminAbuse, User, VideoChannel, VideoDetails, VideoImport } from '../../shared' import { CustomConfig } from '../../shared/models/server/custom-config.model' -import { VideoComment } from '../../shared/models/videos/video-comment.model' +import { VideoComment } from '../../shared/models/videos/comment/video-comment.model' import { CONFIG } from '../initializers/config' import { jsonLoggerFormat, labelFormatter } from './logger' diff --git a/server/helpers/ffprobe-utils.ts b/server/helpers/ffprobe-utils.ts index 40eaafd57..ef2aa3f89 100644 --- a/server/helpers/ffprobe-utils.ts +++ b/server/helpers/ffprobe-utils.ts @@ -1,6 +1,5 @@ import * as ffmpeg from 'fluent-ffmpeg' -import { VideoFileMetadata } from '@shared/models/videos/video-file-metadata' -import { getMaxBitrate, VideoResolution } from '../../shared/models/videos' +import { getMaxBitrate, VideoFileMetadata, VideoResolution } from '../../shared/models/videos' import { CONFIG } from '../initializers/config' import { VIDEO_TRANSCODING_FPS } from '../initializers/constants' import { logger } from './logger' -- cgit v1.2.3 From 16c016e8b1d5ca46343d3363f9a49e24c5d7c944 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 12 May 2021 14:09:04 +0200 Subject: Stricter models typing --- server/helpers/database-utils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'server/helpers') diff --git a/server/helpers/database-utils.ts b/server/helpers/database-utils.ts index f9cb33aca..7befa2c49 100644 --- a/server/helpers/database-utils.ts +++ b/server/helpers/database-utils.ts @@ -68,7 +68,7 @@ function transactionRetryer (func: (err: any, data: T) => any) { }) } -function updateInstanceWithAnother > (instanceToUpdate: Model, baseInstance: Model) { +function updateInstanceWithAnother > (instanceToUpdate: T, baseInstance: U) { const obj = baseInstance.toJSON() for (const key of Object.keys(obj)) { @@ -88,7 +88,7 @@ function afterCommitIfTransaction (t: Transaction, fn: Function) { return fn() } -function deleteNonExistingModels > ( +function deleteNonExistingModels > ( fromDatabase: T[], newModels: T[], t: Transaction -- cgit v1.2.3 From c158a5faabb8ef0bc5d121fda4522d63603e8bc5 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 12 May 2021 14:51:17 +0200 Subject: Refactor a little bit controllers --- server/helpers/custom-validators/misc.ts | 2 +- server/helpers/express-utils.ts | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) (limited to 'server/helpers') diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts index fd3b45804..229e9f03c 100644 --- a/server/helpers/custom-validators/misc.ts +++ b/server/helpers/custom-validators/misc.ts @@ -14,7 +14,7 @@ function isSafePath (p: string) { }) } -function isArray (value: any) { +function isArray (value: any): value is any[] { return Array.isArray(value) } diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts index ede22a3cc..010c6961a 100644 --- a/server/helpers/express-utils.ts +++ b/server/helpers/express-utils.ts @@ -1,13 +1,13 @@ import * as express from 'express' import * as multer from 'multer' +import { extname } from 'path' +import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' +import { CONFIG } from '../initializers/config' import { REMOTE_SCHEME } from '../initializers/constants' +import { isArray } from './custom-validators/misc' import { logger } from './logger' import { deleteFileAndCatch, generateRandomString } from './utils' -import { extname } from 'path' -import { isArray } from './custom-validators/misc' -import { CONFIG } from '../initializers/config' import { getExtFromMimetype } from './video' -import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' function buildNSFWFilter (res?: express.Response, paramNSFW?: string) { if (paramNSFW === 'true') return true @@ -30,21 +30,21 @@ function buildNSFWFilter (res?: express.Response, paramNSFW?: string) { return null } -function cleanUpReqFiles (req: { files: { [fieldname: string]: Express.Multer.File[] } | Express.Multer.File[] }) { - const files = req.files - - if (!files) return +function cleanUpReqFiles ( + req: { files: { [fieldname: string]: Express.Multer.File[] } | Express.Multer.File[] } +) { + const filesObject = req.files + if (!filesObject) return - if (isArray(files)) { - (files as Express.Multer.File[]).forEach(f => deleteFileAndCatch(f.path)) + if (isArray(filesObject)) { + filesObject.forEach(f => deleteFileAndCatch(f.path)) return } - for (const key of Object.keys(files)) { - const file = files[key] + for (const key of Object.keys(filesObject)) { + const files = filesObject[key] - if (isArray(file)) file.forEach(f => deleteFileAndCatch(f.path)) - else deleteFileAndCatch(file.path) + files.forEach(f => deleteFileAndCatch(f.path)) } } -- cgit v1.2.3 From d61fa154af248c8456fb60da21507c767917d804 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 25 May 2021 10:21:36 +0200 Subject: Fix duplicate ffmpeg preset option for live --- server/helpers/ffmpeg-utils.ts | 1 - 1 file changed, 1 deletion(-) (limited to 'server/helpers') diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts index 25d9d4951..e328c49ac 100644 --- a/server/helpers/ffmpeg-utils.ts +++ b/server/helpers/ffmpeg-utils.ts @@ -236,7 +236,6 @@ async function getLiveTranscodingCommand (options: { } ] - command.outputOption('-preset superfast') command.outputOption('-sc_threshold 0') addDefaultEncoderGlobalParams({ command }) -- cgit v1.2.3 From 2539932e16129992a2c0889b4ff527c265a8e2c7 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 27 May 2021 15:59:55 +0200 Subject: Instance homepage support (#4007) * Prepare homepage parsers * Add ability to update instance hompage * Add ability to set homepage as landing page * Add homepage preview in admin * Dynamically update left menu for homepage * Inject home content in homepage * Add videos list and channel miniature custom markup * Remove unused elements in markup service --- server/helpers/markdown.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'server/helpers') diff --git a/server/helpers/markdown.ts b/server/helpers/markdown.ts index 2126bb752..41e57d857 100644 --- a/server/helpers/markdown.ts +++ b/server/helpers/markdown.ts @@ -1,4 +1,6 @@ -import { SANITIZE_OPTIONS, TEXT_WITH_HTML_RULES } from '@shared/core-utils' +import { getSanitizeOptions, TEXT_WITH_HTML_RULES } from '@shared/core-utils' + +const sanitizeOptions = getSanitizeOptions() const sanitizeHtml = require('sanitize-html') const markdownItEmoji = require('markdown-it-emoji/light') @@ -18,7 +20,7 @@ const toSafeHtml = text => { const html = markdownIt.render(textWithLineFeed) // Convert to safe Html - return sanitizeHtml(html, SANITIZE_OPTIONS) + return sanitizeHtml(html, sanitizeOptions) } const mdToPlainText = text => { @@ -28,7 +30,7 @@ const mdToPlainText = text => { const html = markdownIt.render(text) // Convert to safe Html - const safeHtml = sanitizeHtml(html, SANITIZE_OPTIONS) + const safeHtml = sanitizeHtml(html, sanitizeOptions) return safeHtml.replace(/<[^>]+>/g, '') .replace(/\n$/, '') -- cgit v1.2.3