From 60f1f61579947caab1b1d23646cd4e82691b431c Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Thu, 6 May 2021 16:39:17 +0200
Subject: Fix ffmpeg version checker
---
server/helpers/ffmpeg-utils.ts | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
(limited to 'server/helpers')
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts
index 75297df8f..f79b70469 100644
--- a/server/helpers/ffmpeg-utils.ts
+++ b/server/helpers/ffmpeg-utils.ts
@@ -679,10 +679,16 @@ function getFFmpegVersion () {
return execPromise(`${ffmpegPath} -version`)
.then(stdout => {
- const parsed = stdout.match(/ffmpeg version .?(\d+\.\d+\.\d+)/)
+ const parsed = stdout.match(/ffmpeg version .?(\d+\.\d+(\.\d+)?)/)
if (!parsed || !parsed[1]) return rej(new Error(`Could not find ffmpeg version in ${stdout}`))
- return res(parsed[1])
+ // Fix ffmpeg version that does not include patch version (4.4 for example)
+ let version = parsed[1]
+ if (version.match(/^\d+\.\d+/)) {
+ version += '.0'
+ }
+
+ return res(version)
})
.catch(err => rej(err))
})
--
cgit v1.2.3
From a66c2e3252d6cca8958959966f42494ded564023 Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Fri, 7 May 2021 08:59:59 +0200
Subject: Fix remote actor creation date
---
.../helpers/custom-validators/activitypub/actor.ts | 39 ++++++++++------------
1 file changed, 18 insertions(+), 21 deletions(-)
(limited to 'server/helpers')
diff --git a/server/helpers/custom-validators/activitypub/actor.ts b/server/helpers/custom-validators/activitypub/actor.ts
index 877345157..675a7b663 100644
--- a/server/helpers/custom-validators/activitypub/actor.ts
+++ b/server/helpers/custom-validators/activitypub/actor.ts
@@ -1,6 +1,6 @@
import validator from 'validator'
import { CONSTRAINTS_FIELDS } from '../../../initializers/constants'
-import { exists, isArray } from '../misc'
+import { exists, isArray, isDateValid } from '../misc'
import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc'
import { isHostValid } from '../servers'
import { peertubeTruncate } from '@server/helpers/core-utils'
@@ -47,7 +47,21 @@ function isActorPrivateKeyValid (privateKey: string) {
validator.isLength(privateKey, CONSTRAINTS_FIELDS.ACTORS.PRIVATE_KEY)
}
-function isActorObjectValid (actor: any) {
+function isActorFollowingCountValid (value: string) {
+ return exists(value) && validator.isInt('' + value, { min: 0 })
+}
+
+function isActorFollowersCountValid (value: string) {
+ return exists(value) && validator.isInt('' + value, { min: 0 })
+}
+
+function isActorDeleteActivityValid (activity: any) {
+ return isBaseActivityValid(activity, 'Delete')
+}
+
+function sanitizeAndCheckActorObject (actor: any) {
+ normalizeActor(actor)
+
return exists(actor) &&
isActivityPubUrlValid(actor.id) &&
isActorTypeValid(actor.type) &&
@@ -68,24 +82,6 @@ function isActorObjectValid (actor: any) {
(actor.type !== 'Group' || actor.attributedTo.length !== 0)
}
-function isActorFollowingCountValid (value: string) {
- return exists(value) && validator.isInt('' + value, { min: 0 })
-}
-
-function isActorFollowersCountValid (value: string) {
- return exists(value) && validator.isInt('' + value, { min: 0 })
-}
-
-function isActorDeleteActivityValid (activity: any) {
- return isBaseActivityValid(activity, 'Delete')
-}
-
-function sanitizeAndCheckActorObject (object: any) {
- normalizeActor(object)
-
- return isActorObjectValid(object)
-}
-
function normalizeActor (actor: any) {
if (!actor) return
@@ -95,6 +91,8 @@ function normalizeActor (actor: any) {
actor.url = actor.url.href || actor.url.url
}
+ if (!isDateValid(actor.published)) actor.published = undefined
+
if (actor.summary && typeof actor.summary === 'string') {
actor.summary = peertubeTruncate(actor.summary, { length: CONSTRAINTS_FIELDS.USERS.DESCRIPTION.max })
@@ -135,7 +133,6 @@ export {
isActorPublicKeyValid,
isActorPreferredUsernameValid,
isActorPrivateKeyValid,
- isActorObjectValid,
isActorFollowingCountValid,
isActorFollowersCountValid,
isActorDeleteActivityValid,
--
cgit v1.2.3
From 1ff9f1cda3f8283241776cd8344e6671742b25f7 Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Fri, 7 May 2021 14:23:43 +0200
Subject: Fix ffmpeg version parsing
---
server/helpers/ffmpeg-utils.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
(limited to 'server/helpers')
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts
index f79b70469..25d9d4951 100644
--- a/server/helpers/ffmpeg-utils.ts
+++ b/server/helpers/ffmpeg-utils.ts
@@ -684,7 +684,7 @@ function getFFmpegVersion () {
// Fix ffmpeg version that does not include patch version (4.4 for example)
let version = parsed[1]
- if (version.match(/^\d+\.\d+/)) {
+ if (version.match(/^\d+\.\d+$/)) {
version += '.0'
}
--
cgit v1.2.3
From e024fd6a7494b37251da1d59470324305cdb4129 Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Fri, 7 May 2021 17:14:39 +0200
Subject: Update channel updatedAt when uploading a video
---
server/helpers/database-utils.ts | 18 ++++++++++++++++--
1 file changed, 16 insertions(+), 2 deletions(-)
(limited to 'server/helpers')
diff --git a/server/helpers/database-utils.ts b/server/helpers/database-utils.ts
index 2b916efc2..f9cb33aca 100644
--- a/server/helpers/database-utils.ts
+++ b/server/helpers/database-utils.ts
@@ -1,8 +1,9 @@
import * as retry from 'async/retry'
import * as Bluebird from 'bluebird'
+import { QueryTypes, Transaction } from 'sequelize'
import { Model } from 'sequelize-typescript'
+import { sequelizeTypescript } from '@server/initializers/database'
import { logger } from './logger'
-import { Transaction } from 'sequelize'
function retryTransactionWrapper <T, A, B, C, D> (
functionToRetry: (arg1: A, arg2: B, arg3: C, arg4: D) => Promise<T> | Bluebird<T>,
@@ -96,6 +97,18 @@ function deleteNonExistingModels <T extends { hasSameUniqueKeysThan (other: T):
.map(f => f.destroy({ transaction: t }))
}
+// Sequelize always skip the update if we only update updatedAt field
+function setAsUpdated (table: string, id: number, transaction?: Transaction) {
+ return sequelizeTypescript.query(
+ `UPDATE "${table}" SET "updatedAt" = :updatedAt WHERE id = :id`,
+ {
+ replacements: { table, id, updatedAt: new Date() },
+ type: QueryTypes.UPDATE,
+ transaction
+ }
+ )
+}
+
// ---------------------------------------------------------------------------
export {
@@ -104,5 +117,6 @@ export {
transactionRetryer,
updateInstanceWithAnother,
afterCommitIfTransaction,
- deleteNonExistingModels
+ deleteNonExistingModels,
+ setAsUpdated
}
--
cgit v1.2.3
From f6d6e7f861189a4446f406efb775a29688764b48 Mon Sep 17 00:00:00 2001
From: kontrollanten <6680299+kontrollanten@users.noreply.github.com>
Date: Mon, 10 May 2021 11:13:41 +0200
Subject: Resumable video uploads (#3933)
* WIP: resumable video uploads
relates to #324
* fix review comments
* video upload: error handling
* fix audio upload
* fixes after self review
* Update server/controllers/api/videos/index.ts
Co-authored-by: Rigel Kent <par@rigelk.eu>
* Update server/middlewares/validators/videos/videos.ts
Co-authored-by: Rigel Kent <par@rigelk.eu>
* Update server/controllers/api/videos/index.ts
Co-authored-by: Rigel Kent <par@rigelk.eu>
* update after code review
* refactor upload route
- restore multipart upload route
- move resumable to dedicated upload-resumable route
- move checks to middleware
- do not leak internal fs structure in response
* fix yarn.lock upon rebase
* factorize addVideo for reuse in both endpoints
* add resumable upload API to openapi spec
* add initial test and test helper for resumable upload
* typings for videoAddResumable middleware
* avoid including aws and google packages via node-uploadx, by only including uploadx/core
* rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio
* add video-upload-tmp-folder-cleaner job
* stronger typing of video upload middleware
* reduce dependency to @uploadx/core
* add audio upload test
* refactor resumable uploads cleanup from job to scheduler
* refactor resumable uploads scheduler to compare to last execution time
* make resumable upload validator to always cleanup on failure
* move legacy upload request building outside of uploadVideo test helper
* filter upload-resumable middlewares down to POST, PUT, DELETE
also begin to type metadata
* merge add duration functions
* stronger typings and documentation for uploadx behaviour, move init validator up
* refactor(client/video-edit): options > uploadxOptions
* refactor(client/video-edit): remove obsolete else
* scheduler/remove-dangling-resum: rename tag
* refactor(server/video): add UploadVideoFiles type
* refactor(mw/validators): restructure eslint disable
* refactor(mw/validators/videos): rename import
* refactor(client/vid-upload): rename html elem id
* refactor(sched/remove-dangl): move fn to method
* refactor(mw/async): add method typing
* refactor(mw/vali/video): double quote > single
* refactor(server/upload-resum): express use > all
* proper http methud enum server/middlewares/async.ts
* properly type http methods
* factorize common video upload validation steps
* add check for maximum partially uploaded file size
* fix audioBg use
* fix extname(filename) in addVideo
* document parameters for uploadx's resumable protocol
* clear META files in scheduler
* last audio refactor before cramming preview in the initial POST form data
* refactor as mulitpart/form-data initial post request
this allows preview/thumbnail uploads alongside the initial request,
and cleans up the upload form
* Add more tests for resumable uploads
* Refactor remove dangling resumable uploads
* Prepare changelog
* Add more resumable upload tests
* Remove user quota check for resumable uploads
* Fix upload error handler
* Update nginx template for upload-resumable
* Cleanup comment
* Remove unused express methods
* Prefer to use got instead of raw http
* Don't retry on error 500
Co-authored-by: Rigel Kent <par@rigelk.eu>
Co-authored-by: Rigel Kent <sendmemail@rigelk.eu>
Co-authored-by: Chocobozzz <me@florianbigard.com>
---
server/helpers/custom-validators/misc.ts | 5 +++--
server/helpers/custom-validators/videos.ts | 9 +++++----
server/helpers/express-utils.ts | 8 ++++----
server/helpers/upload.ts | 21 +++++++++++++++++++++
server/helpers/utils.ts | 4 ++--
5 files changed, 35 insertions(+), 12 deletions(-)
create mode 100644 server/helpers/upload.ts
(limited to 'server/helpers')
diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts
index effdd98cb..fd3b45804 100644
--- a/server/helpers/custom-validators/misc.ts
+++ b/server/helpers/custom-validators/misc.ts
@@ -1,6 +1,7 @@
import 'multer'
-import validator from 'validator'
+import { UploadFilesForCheck } from 'express'
import { sep } from 'path'
+import validator from 'validator'
function exists (value: any) {
return value !== undefined && value !== null
@@ -108,7 +109,7 @@ function isFileFieldValid (
}
function isFileMimeTypeValid (
- files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[],
+ files: UploadFilesForCheck,
mimeTypeRegex: string,
field: string,
optional = false
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts
index 87966798f..b33e088eb 100644
--- a/server/helpers/custom-validators/videos.ts
+++ b/server/helpers/custom-validators/videos.ts
@@ -1,4 +1,6 @@
+import { UploadFilesForCheck } from 'express'
import { values } from 'lodash'
+import * as magnetUtil from 'magnet-uri'
import validator from 'validator'
import { VideoFilter, VideoPrivacy, VideoRateType } from '../../../shared'
import {
@@ -6,13 +8,12 @@ import {
MIMETYPES,
VIDEO_CATEGORIES,
VIDEO_LICENCES,
+ VIDEO_LIVE,
VIDEO_PRIVACIES,
VIDEO_RATE_TYPES,
- VIDEO_STATES,
- VIDEO_LIVE
+ VIDEO_STATES
} from '../../initializers/constants'
import { exists, isArray, isDateValid, isFileMimeTypeValid, isFileValid } from './misc'
-import * as magnetUtil from 'magnet-uri'
const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS
@@ -81,7 +82,7 @@ function isVideoFileExtnameValid (value: string) {
return exists(value) && (value === VIDEO_LIVE.EXTENSION || MIMETYPES.VIDEO.EXT_MIMETYPE[value] !== undefined)
}
-function isVideoFileMimeTypeValid (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) {
+function isVideoFileMimeTypeValid (files: UploadFilesForCheck) {
return isFileMimeTypeValid(files, MIMETYPES.VIDEO.MIMETYPES_REGEX, 'videofile')
}
diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts
index c0d3f8f32..ede22a3cc 100644
--- a/server/helpers/express-utils.ts
+++ b/server/helpers/express-utils.ts
@@ -2,7 +2,7 @@ import * as express from 'express'
import * as multer from 'multer'
import { REMOTE_SCHEME } from '../initializers/constants'
import { logger } from './logger'
-import { deleteFileAsync, generateRandomString } from './utils'
+import { deleteFileAndCatch, generateRandomString } from './utils'
import { extname } from 'path'
import { isArray } from './custom-validators/misc'
import { CONFIG } from '../initializers/config'
@@ -36,15 +36,15 @@ function cleanUpReqFiles (req: { files: { [fieldname: string]: Express.Multer.Fi
if (!files) return
if (isArray(files)) {
- (files as Express.Multer.File[]).forEach(f => deleteFileAsync(f.path))
+ (files as Express.Multer.File[]).forEach(f => deleteFileAndCatch(f.path))
return
}
for (const key of Object.keys(files)) {
const file = files[key]
- if (isArray(file)) file.forEach(f => deleteFileAsync(f.path))
- else deleteFileAsync(file.path)
+ if (isArray(file)) file.forEach(f => deleteFileAndCatch(f.path))
+ else deleteFileAndCatch(file.path)
}
}
diff --git a/server/helpers/upload.ts b/server/helpers/upload.ts
new file mode 100644
index 000000000..030a6b7d5
--- /dev/null
+++ b/server/helpers/upload.ts
@@ -0,0 +1,21 @@
+import { METAFILE_EXTNAME } from '@uploadx/core'
+import { remove } from 'fs-extra'
+import { join } from 'path'
+import { RESUMABLE_UPLOAD_DIRECTORY } from '../initializers/constants'
+
+function getResumableUploadPath (filename?: string) {
+ if (filename) return join(RESUMABLE_UPLOAD_DIRECTORY, filename)
+
+ return RESUMABLE_UPLOAD_DIRECTORY
+}
+
+function deleteResumableUploadMetaFile (filepath: string) {
+ return remove(filepath + METAFILE_EXTNAME)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+ getResumableUploadPath,
+ deleteResumableUploadMetaFile
+}
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts
index 0545e8996..6c95a43b6 100644
--- a/server/helpers/utils.ts
+++ b/server/helpers/utils.ts
@@ -6,7 +6,7 @@ import { CONFIG } from '../initializers/config'
import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils'
import { logger } from './logger'
-function deleteFileAsync (path: string) {
+function deleteFileAndCatch (path: string) {
remove(path)
.catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err }))
}
@@ -83,7 +83,7 @@ function getUUIDFromFilename (filename: string) {
// ---------------------------------------------------------------------------
export {
- deleteFileAsync,
+ deleteFileAndCatch,
generateRandomString,
getFormattedObjects,
getSecureTorrentName,
--
cgit v1.2.3
From 1bcb03a100d172903b877d6a0e4ed11d63b14f3d Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Tue, 11 May 2021 10:54:05 +0200
Subject: Use a class for youtube-dl
---
server/helpers/youtube-dl.ts | 553 +++++++++++++++++++++----------------------
1 file changed, 275 insertions(+), 278 deletions(-)
(limited to 'server/helpers')
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
index fac3da6ba..d003ea3cf 100644
--- a/server/helpers/youtube-dl.ts
+++ b/server/helpers/youtube-dl.ts
@@ -6,7 +6,6 @@ import { CONFIG } from '@server/initializers/config'
import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes'
import { VideoResolution } from '../../shared/models/videos'
import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants'
-import { getEnabledResolutions } from '../lib/video-transcoding'
import { peertubeTruncate, pipelinePromise, root } from './core-utils'
import { isVideoFileExtnameValid } from './custom-validators/videos'
import { logger } from './logger'
@@ -35,361 +34,359 @@ const processOptions = {
maxBuffer: 1024 * 1024 * 10 // 10MB
}
-function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo> {
- return new Promise<YoutubeDLInfo>((res, rej) => {
- let args = opts || [ '-j', '--flat-playlist' ]
+class YoutubeDL {
- if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) {
- args.push('--force-ipv4')
- }
+ constructor (private readonly url: string = '', private readonly enabledResolutions: number[] = []) {
- args = wrapWithProxyOptions(args)
- args = [ '-f', getYoutubeDLVideoFormat() ].concat(args)
+ }
- safeGetYoutubeDL()
- .then(youtubeDL => {
- youtubeDL.getInfo(url, args, processOptions, (err, info) => {
- if (err) return rej(err)
- if (info.is_live === true) return rej(new Error('Cannot download a live streaming.'))
+ getYoutubeDLInfo (opts?: string[]): Promise<YoutubeDLInfo> {
+ return new Promise<YoutubeDLInfo>((res, rej) => {
+ let args = opts || [ '-j', '--flat-playlist' ]
- const obj = buildVideoInfo(normalizeObject(info))
- if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video'
+ if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) {
+ args.push('--force-ipv4')
+ }
- return res(obj)
- })
- })
- .catch(err => rej(err))
- })
-}
+ args = this.wrapWithProxyOptions(args)
+ args = [ '-f', this.getYoutubeDLVideoFormat() ].concat(args)
-function getYoutubeDLSubs (url: string, opts?: object): Promise<YoutubeDLSubs> {
- return new Promise<YoutubeDLSubs>((res, rej) => {
- const cwd = CONFIG.STORAGE.TMP_DIR
- const options = opts || { all: true, format: 'vtt', cwd }
-
- safeGetYoutubeDL()
- .then(youtubeDL => {
- youtubeDL.getSubs(url, options, (err, files) => {
- if (err) return rej(err)
- if (!files) return []
-
- logger.debug('Get subtitles from youtube dl.', { url, files })
-
- const subtitles = files.reduce((acc, filename) => {
- const matched = filename.match(/\.([a-z]{2})(-[a-z]+)?\.(vtt|ttml)/i)
- if (!matched || !matched[1]) return acc
-
- return [
- ...acc,
- {
- language: matched[1],
- path: join(cwd, filename),
- filename
- }
- ]
- }, [])
+ YoutubeDL.safeGetYoutubeDL()
+ .then(youtubeDL => {
+ youtubeDL.getInfo(this.url, args, processOptions, (err, info) => {
+ if (err) return rej(err)
+ if (info.is_live === true) return rej(new Error('Cannot download a live streaming.'))
- return res(subtitles)
+ const obj = this.buildVideoInfo(this.normalizeObject(info))
+ if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video'
+
+ return res(obj)
+ })
})
- })
- .catch(err => rej(err))
- })
-}
+ .catch(err => rej(err))
+ })
+ }
-function getYoutubeDLVideoFormat () {
- /**
- * list of format selectors in order or preference
- * see https://github.com/ytdl-org/youtube-dl#format-selection
- *
- * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope
- * of being able to do a "quick-transcode"
- * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9)
- * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback
- *
- * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499
- **/
- const enabledResolutions = getEnabledResolutions('vod')
- const resolution = enabledResolutions.length === 0
- ? VideoResolution.H_720P
- : Math.max(...enabledResolutions)
-
- return [
- `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1
- `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2
- `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]`, // case #3
- `bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio`,
- 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
- 'best' // Ultimate fallback
- ].join('/')
-}
+ getYoutubeDLSubs (opts?: object): Promise<YoutubeDLSubs> {
+ return new Promise<YoutubeDLSubs>((res, rej) => {
+ const cwd = CONFIG.STORAGE.TMP_DIR
+ const options = opts || { all: true, format: 'vtt', cwd }
+
+ YoutubeDL.safeGetYoutubeDL()
+ .then(youtubeDL => {
+ youtubeDL.getSubs(this.url, options, (err, files) => {
+ if (err) return rej(err)
+ if (!files) return []
+
+ logger.debug('Get subtitles from youtube dl.', { url: this.url, files })
+
+ const subtitles = files.reduce((acc, filename) => {
+ const matched = filename.match(/\.([a-z]{2})(-[a-z]+)?\.(vtt|ttml)/i)
+ if (!matched || !matched[1]) return acc
+
+ return [
+ ...acc,
+ {
+ language: matched[1],
+ path: join(cwd, filename),
+ filename
+ }
+ ]
+ }, [])
+
+ return res(subtitles)
+ })
+ })
+ .catch(err => rej(err))
+ })
+ }
-function downloadYoutubeDLVideo (url: string, fileExt: string, timeout: number) {
- // Leave empty the extension, youtube-dl will add it
- const pathWithoutExtension = generateVideoImportTmpPath(url, '')
+ getYoutubeDLVideoFormat () {
+ /**
+ * list of format selectors in order or preference
+ * see https://github.com/ytdl-org/youtube-dl#format-selection
+ *
+ * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope
+ * of being able to do a "quick-transcode"
+ * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9)
+ * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback
+ *
+ * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499
+ **/
+ const resolution = this.enabledResolutions.length === 0
+ ? VideoResolution.H_720P
+ : Math.max(...this.enabledResolutions)
+
+ return [
+ `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1
+ `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2
+ `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]`, // case #3
+ `bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio`,
+ 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
+ 'best' // Ultimate fallback
+ ].join('/')
+ }
- let timer
+ downloadYoutubeDLVideo (fileExt: string, timeout: number) {
+ // Leave empty the extension, youtube-dl will add it
+ const pathWithoutExtension = generateVideoImportTmpPath(this.url, '')
- logger.info('Importing youtubeDL video %s to %s', url, pathWithoutExtension)
+ let timer
- let options = [ '-f', getYoutubeDLVideoFormat(), '-o', pathWithoutExtension ]
- options = wrapWithProxyOptions(options)
+ logger.info('Importing youtubeDL video %s to %s', this.url, pathWithoutExtension)
- if (process.env.FFMPEG_PATH) {
- options = options.concat([ '--ffmpeg-location', process.env.FFMPEG_PATH ])
- }
+ let options = [ '-f', this.getYoutubeDLVideoFormat(), '-o', pathWithoutExtension ]
+ options = this.wrapWithProxyOptions(options)
- logger.debug('YoutubeDL options for %s.', url, { options })
+ if (process.env.FFMPEG_PATH) {
+ options = options.concat([ '--ffmpeg-location', process.env.FFMPEG_PATH ])
+ }
- return new Promise<string>((res, rej) => {
- safeGetYoutubeDL()
- .then(youtubeDL => {
- youtubeDL.exec(url, options, processOptions, async err => {
- clearTimeout(timer)
+ logger.debug('YoutubeDL options for %s.', this.url, { options })
- try {
- // If youtube-dl did not guess an extension for our file, just use .mp4 as default
- if (await pathExists(pathWithoutExtension)) {
- await move(pathWithoutExtension, pathWithoutExtension + '.mp4')
- }
+ return new Promise<string>((res, rej) => {
+ YoutubeDL.safeGetYoutubeDL()
+ .then(youtubeDL => {
+ youtubeDL.exec(this.url, options, processOptions, async err => {
+ clearTimeout(timer)
+
+ try {
+ // If youtube-dl did not guess an extension for our file, just use .mp4 as default
+ if (await pathExists(pathWithoutExtension)) {
+ await move(pathWithoutExtension, pathWithoutExtension + '.mp4')
+ }
- const path = await guessVideoPathWithExtension(pathWithoutExtension, fileExt)
+ const path = await this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
- if (err) {
- remove(path)
- .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err }))
+ if (err) {
+ remove(path)
+ .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err }))
+ return rej(err)
+ }
+
+ return res(path)
+ } catch (err) {
return rej(err)
}
-
- return res(path)
- } catch (err) {
- return rej(err)
- }
+ })
+
+ timer = setTimeout(() => {
+ const err = new Error('YoutubeDL download timeout.')
+
+ this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
+ .then(path => remove(path))
+ .finally(() => rej(err))
+ .catch(err => {
+ logger.error('Cannot remove file in youtubeDL timeout.', { err })
+ return rej(err)
+ })
+ }, timeout)
})
+ .catch(err => rej(err))
+ })
+ }
- timer = setTimeout(() => {
- const err = new Error('YoutubeDL download timeout.')
+ buildOriginallyPublishedAt (obj: any) {
+ let originallyPublishedAt: Date = null
- guessVideoPathWithExtension(pathWithoutExtension, fileExt)
- .then(path => remove(path))
- .finally(() => rej(err))
- .catch(err => {
- logger.error('Cannot remove file in youtubeDL timeout.', { err })
- return rej(err)
- })
- }, timeout)
- })
- .catch(err => rej(err))
- })
-}
-
-// Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js
-// We rewrote it to avoid sync calls
-async function updateYoutubeDLBinary () {
- logger.info('Updating youtubeDL binary.')
+ const uploadDateMatcher = /^(\d{4})(\d{2})(\d{2})$/.exec(obj.upload_date)
+ if (uploadDateMatcher) {
+ originallyPublishedAt = new Date()
+ originallyPublishedAt.setHours(0, 0, 0, 0)
- const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin')
- const bin = join(binDirectory, 'youtube-dl')
- const detailsPath = join(binDirectory, 'details')
- const url = process.env.YOUTUBE_DL_DOWNLOAD_HOST || 'https://yt-dl.org/downloads/latest/youtube-dl'
+ const year = parseInt(uploadDateMatcher[1], 10)
+ // Month starts from 0
+ const month = parseInt(uploadDateMatcher[2], 10) - 1
+ const day = parseInt(uploadDateMatcher[3], 10)
- await ensureDir(binDirectory)
+ originallyPublishedAt.setFullYear(year, month, day)
+ }
- try {
- const result = await got(url, { followRedirect: false })
+ return originallyPublishedAt
+ }
- if (result.statusCode !== HttpStatusCode.FOUND_302) {
- logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
- return
+ private async guessVideoPathWithExtension (tmpPath: string, sourceExt: string) {
+ if (!isVideoFileExtnameValid(sourceExt)) {
+ throw new Error('Invalid video extension ' + sourceExt)
}
- const newUrl = result.headers.location
- const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(newUrl)[1]
+ const extensions = [ sourceExt, '.mp4', '.mkv', '.webm' ]
- const downloadFileStream = got.stream(newUrl)
- const writeStream = createWriteStream(bin, { mode: 493 })
+ for (const extension of extensions) {
+ const path = tmpPath + extension
- await pipelinePromise(
- downloadFileStream,
- writeStream
- )
-
- const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
- await writeFile(detailsPath, details, { encoding: 'utf8' })
+ if (await pathExists(path)) return path
+ }
- logger.info('youtube-dl updated to version %s.', newVersion)
- } catch (err) {
- logger.error('Cannot update youtube-dl.', { err })
+ throw new Error('Cannot guess path of ' + tmpPath)
}
-}
-async function safeGetYoutubeDL () {
- let youtubeDL
+ private normalizeObject (obj: any) {
+ const newObj: any = {}
- try {
- youtubeDL = require('youtube-dl')
- } catch (e) {
- // Download binary
- await updateYoutubeDLBinary()
- youtubeDL = require('youtube-dl')
- }
+ for (const key of Object.keys(obj)) {
+ // Deprecated key
+ if (key === 'resolution') continue
- return youtubeDL
-}
+ const value = obj[key]
-function buildOriginallyPublishedAt (obj: any) {
- let originallyPublishedAt: Date = null
+ if (typeof value === 'string') {
+ newObj[key] = value.normalize()
+ } else {
+ newObj[key] = value
+ }
+ }
- const uploadDateMatcher = /^(\d{4})(\d{2})(\d{2})$/.exec(obj.upload_date)
- if (uploadDateMatcher) {
- originallyPublishedAt = new Date()
- originallyPublishedAt.setHours(0, 0, 0, 0)
+ return newObj
+ }
- const year = parseInt(uploadDateMatcher[1], 10)
- // Month starts from 0
- const month = parseInt(uploadDateMatcher[2], 10) - 1
- const day = parseInt(uploadDateMatcher[3], 10)
+ private buildVideoInfo (obj: any): YoutubeDLInfo {
+ return {
+ name: this.titleTruncation(obj.title),
+ description: this.descriptionTruncation(obj.description),
+ category: this.getCategory(obj.categories),
+ licence: this.getLicence(obj.license),
+ language: this.getLanguage(obj.language),
+ nsfw: this.isNSFW(obj),
+ tags: this.getTags(obj.tags),
+ thumbnailUrl: obj.thumbnail || undefined,
+ originallyPublishedAt: this.buildOriginallyPublishedAt(obj),
+ ext: obj.ext
+ }
+ }
- originallyPublishedAt.setFullYear(year, month, day)
+ private titleTruncation (title: string) {
+ return peertubeTruncate(title, {
+ length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
+ separator: /,? +/,
+ omission: ' […]'
+ })
}
- return originallyPublishedAt
-}
+ private descriptionTruncation (description: string) {
+ if (!description || description.length < CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.min) return undefined
-// ---------------------------------------------------------------------------
+ return peertubeTruncate(description, {
+ length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max,
+ separator: /,? +/,
+ omission: ' […]'
+ })
+ }
-export {
- updateYoutubeDLBinary,
- getYoutubeDLVideoFormat,
- downloadYoutubeDLVideo,
- getYoutubeDLSubs,
- getYoutubeDLInfo,
- safeGetYoutubeDL,
- buildOriginallyPublishedAt
-}
+ private isNSFW (info: any) {
+ return info.age_limit && info.age_limit >= 16
+ }
-// ---------------------------------------------------------------------------
+ private getTags (tags: any) {
+ if (Array.isArray(tags) === false) return []
-async function guessVideoPathWithExtension (tmpPath: string, sourceExt: string) {
- if (!isVideoFileExtnameValid(sourceExt)) {
- throw new Error('Invalid video extension ' + sourceExt)
+ return tags
+ .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
+ .map(t => t.normalize())
+ .slice(0, 5)
}
- const extensions = [ sourceExt, '.mp4', '.mkv', '.webm' ]
+ private getLicence (licence: string) {
+ if (!licence) return undefined
- for (const extension of extensions) {
- const path = tmpPath + extension
+ if (licence.includes('Creative Commons Attribution')) return 1
- if (await pathExists(path)) return path
- }
+ for (const key of Object.keys(VIDEO_LICENCES)) {
+ const peertubeLicence = VIDEO_LICENCES[key]
+ if (peertubeLicence.toLowerCase() === licence.toLowerCase()) return parseInt(key, 10)
+ }
- throw new Error('Cannot guess path of ' + tmpPath)
-}
+ return undefined
+ }
-function normalizeObject (obj: any) {
- const newObj: any = {}
+ private getCategory (categories: string[]) {
+ if (!categories) return undefined
- for (const key of Object.keys(obj)) {
- // Deprecated key
- if (key === 'resolution') continue
+ const categoryString = categories[0]
+ if (!categoryString || typeof categoryString !== 'string') return undefined
- const value = obj[key]
+ if (categoryString === 'News & Politics') return 11
- if (typeof value === 'string') {
- newObj[key] = value.normalize()
- } else {
- newObj[key] = value
+ for (const key of Object.keys(VIDEO_CATEGORIES)) {
+ const category = VIDEO_CATEGORIES[key]
+ if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10)
}
- }
- return newObj
-}
-
-function buildVideoInfo (obj: any): YoutubeDLInfo {
- return {
- name: titleTruncation(obj.title),
- description: descriptionTruncation(obj.description),
- category: getCategory(obj.categories),
- licence: getLicence(obj.license),
- language: getLanguage(obj.language),
- nsfw: isNSFW(obj),
- tags: getTags(obj.tags),
- thumbnailUrl: obj.thumbnail || undefined,
- originallyPublishedAt: buildOriginallyPublishedAt(obj),
- ext: obj.ext
+ return undefined
}
-}
-function titleTruncation (title: string) {
- return peertubeTruncate(title, {
- length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
- separator: /,? +/,
- omission: ' […]'
- })
-}
+ private getLanguage (language: string) {
+ return VIDEO_LANGUAGES[language] ? language : undefined
+ }
-function descriptionTruncation (description: string) {
- if (!description || description.length < CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.min) return undefined
+ private wrapWithProxyOptions (options: string[]) {
+ if (CONFIG.IMPORT.VIDEOS.HTTP.PROXY.ENABLED) {
+ logger.debug('Using proxy for YoutubeDL')
- return peertubeTruncate(description, {
- length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max,
- separator: /,? +/,
- omission: ' […]'
- })
-}
+ return [ '--proxy', CONFIG.IMPORT.VIDEOS.HTTP.PROXY.URL ].concat(options)
+ }
-function isNSFW (info: any) {
- return info.age_limit && info.age_limit >= 16
-}
+ return options
+ }
-function getTags (tags: any) {
- if (Array.isArray(tags) === false) return []
+ // Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js
+ // We rewrote it to avoid sync calls
+ static async updateYoutubeDLBinary () {
+ logger.info('Updating youtubeDL binary.')
- return tags
- .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
- .map(t => t.normalize())
- .slice(0, 5)
-}
+ const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin')
+ const bin = join(binDirectory, 'youtube-dl')
+ const detailsPath = join(binDirectory, 'details')
+ const url = process.env.YOUTUBE_DL_DOWNLOAD_HOST || 'https://yt-dl.org/downloads/latest/youtube-dl'
-function getLicence (licence: string) {
- if (!licence) return undefined
+ await ensureDir(binDirectory)
- if (licence.includes('Creative Commons Attribution')) return 1
+ try {
+ const result = await got(url, { followRedirect: false })
- for (const key of Object.keys(VIDEO_LICENCES)) {
- const peertubeLicence = VIDEO_LICENCES[key]
- if (peertubeLicence.toLowerCase() === licence.toLowerCase()) return parseInt(key, 10)
- }
+ if (result.statusCode !== HttpStatusCode.FOUND_302) {
+ logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
+ return
+ }
- return undefined
-}
+ const newUrl = result.headers.location
+ const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(newUrl)[1]
-function getCategory (categories: string[]) {
- if (!categories) return undefined
+ const downloadFileStream = got.stream(newUrl)
+ const writeStream = createWriteStream(bin, { mode: 493 })
- const categoryString = categories[0]
- if (!categoryString || typeof categoryString !== 'string') return undefined
+ await pipelinePromise(
+ downloadFileStream,
+ writeStream
+ )
- if (categoryString === 'News & Politics') return 11
+ const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
+ await writeFile(detailsPath, details, { encoding: 'utf8' })
- for (const key of Object.keys(VIDEO_CATEGORIES)) {
- const category = VIDEO_CATEGORIES[key]
- if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10)
+ logger.info('youtube-dl updated to version %s.', newVersion)
+ } catch (err) {
+ logger.error('Cannot update youtube-dl.', { err })
+ }
}
- return undefined
-}
-
-function getLanguage (language: string) {
- return VIDEO_LANGUAGES[language] ? language : undefined
-}
+ static async safeGetYoutubeDL () {
+ let youtubeDL
-function wrapWithProxyOptions (options: string[]) {
- if (CONFIG.IMPORT.VIDEOS.HTTP.PROXY.ENABLED) {
- logger.debug('Using proxy for YoutubeDL')
+ try {
+ youtubeDL = require('youtube-dl')
+ } catch (e) {
+ // Download binary
+ await this.updateYoutubeDLBinary()
+ youtubeDL = require('youtube-dl')
+ }
- return [ '--proxy', CONFIG.IMPORT.VIDEOS.HTTP.PROXY.URL ].concat(options)
+ return youtubeDL
}
+}
+
+// ---------------------------------------------------------------------------
- return options
+export {
+ YoutubeDL
}
--
cgit v1.2.3
From 7d9ba5c08999c6482f0bc5e0c09c6f55b7724090 Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Tue, 11 May 2021 11:15:29 +0200
Subject: Cleanup models directory organization
---
server/helpers/actor.ts | 2 +-
server/helpers/middlewares/accounts.ts | 2 +-
server/helpers/signup.ts | 2 +-
server/helpers/webfinger.ts | 6 +++---
4 files changed, 6 insertions(+), 6 deletions(-)
(limited to 'server/helpers')
diff --git a/server/helpers/actor.ts b/server/helpers/actor.ts
index a60d3ed5d..5f742505b 100644
--- a/server/helpers/actor.ts
+++ b/server/helpers/actor.ts
@@ -1,5 +1,5 @@
-import { ActorModel } from '../models/activitypub/actor'
+import { ActorModel } from '../models/actor/actor'
import { MActorAccountChannelId, MActorFull } from '../types/models'
type ActorFetchByUrlType = 'all' | 'association-ids'
diff --git a/server/helpers/middlewares/accounts.ts b/server/helpers/middlewares/accounts.ts
index 13ae6cdf4..5addd3e1a 100644
--- a/server/helpers/middlewares/accounts.ts
+++ b/server/helpers/middlewares/accounts.ts
@@ -1,5 +1,5 @@
import { Response } from 'express'
-import { UserModel } from '@server/models/account/user'
+import { UserModel } from '@server/models/user/user'
import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
import { AccountModel } from '../../models/account/account'
import { MAccountDefault } from '../../types/models'
diff --git a/server/helpers/signup.ts b/server/helpers/signup.ts
index ed872539b..8fa81e601 100644
--- a/server/helpers/signup.ts
+++ b/server/helpers/signup.ts
@@ -1,4 +1,4 @@
-import { UserModel } from '../models/account/user'
+import { UserModel } from '../models/user/user'
import * as ipaddr from 'ipaddr.js'
import { CONFIG } from '../initializers/config'
diff --git a/server/helpers/webfinger.ts b/server/helpers/webfinger.ts
index da7e88077..33367f651 100644
--- a/server/helpers/webfinger.ts
+++ b/server/helpers/webfinger.ts
@@ -1,10 +1,10 @@
import * as WebFinger from 'webfinger.js'
import { WebFingerData } from '../../shared'
-import { ActorModel } from '../models/activitypub/actor'
-import { isTestInstance } from './core-utils'
-import { isActivityPubUrlValid } from './custom-validators/activitypub/misc'
import { WEBSERVER } from '../initializers/constants'
+import { ActorModel } from '../models/actor/actor'
import { MActorFull } from '../types/models'
+import { isTestInstance } from './core-utils'
+import { isActivityPubUrlValid } from './custom-validators/activitypub/misc'
const webfinger = new WebFinger({
webfist_fallback: false,
--
cgit v1.2.3
From 2b02c520e66ea452687cab39401b371711caa9ed Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Tue, 11 May 2021 11:27:40 +0200
Subject: Cleanup shared models
---
server/helpers/audit-logger.ts | 2 +-
server/helpers/ffprobe-utils.ts | 3 +--
2 files changed, 2 insertions(+), 3 deletions(-)
(limited to 'server/helpers')
diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts
index 6aae5e821..884bd187d 100644
--- a/server/helpers/audit-logger.ts
+++ b/server/helpers/audit-logger.ts
@@ -7,7 +7,7 @@ import * as winston from 'winston'
import { AUDIT_LOG_FILENAME } from '@server/initializers/constants'
import { AdminAbuse, User, VideoChannel, VideoDetails, VideoImport } from '../../shared'
import { CustomConfig } from '../../shared/models/server/custom-config.model'
-import { VideoComment } from '../../shared/models/videos/video-comment.model'
+import { VideoComment } from '../../shared/models/videos/comment/video-comment.model'
import { CONFIG } from '../initializers/config'
import { jsonLoggerFormat, labelFormatter } from './logger'
diff --git a/server/helpers/ffprobe-utils.ts b/server/helpers/ffprobe-utils.ts
index 40eaafd57..ef2aa3f89 100644
--- a/server/helpers/ffprobe-utils.ts
+++ b/server/helpers/ffprobe-utils.ts
@@ -1,6 +1,5 @@
import * as ffmpeg from 'fluent-ffmpeg'
-import { VideoFileMetadata } from '@shared/models/videos/video-file-metadata'
-import { getMaxBitrate, VideoResolution } from '../../shared/models/videos'
+import { getMaxBitrate, VideoFileMetadata, VideoResolution } from '../../shared/models/videos'
import { CONFIG } from '../initializers/config'
import { VIDEO_TRANSCODING_FPS } from '../initializers/constants'
import { logger } from './logger'
--
cgit v1.2.3
From 16c016e8b1d5ca46343d3363f9a49e24c5d7c944 Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Wed, 12 May 2021 14:09:04 +0200
Subject: Stricter models typing
---
server/helpers/database-utils.ts | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
(limited to 'server/helpers')
diff --git a/server/helpers/database-utils.ts b/server/helpers/database-utils.ts
index f9cb33aca..7befa2c49 100644
--- a/server/helpers/database-utils.ts
+++ b/server/helpers/database-utils.ts
@@ -68,7 +68,7 @@ function transactionRetryer <T> (func: (err: any, data: T) => any) {
})
}
-function updateInstanceWithAnother <T extends Model<T>> (instanceToUpdate: Model<T>, baseInstance: Model<T>) {
+function updateInstanceWithAnother <M, T extends U, U extends Model<M>> (instanceToUpdate: T, baseInstance: U) {
const obj = baseInstance.toJSON()
for (const key of Object.keys(obj)) {
@@ -88,7 +88,7 @@ function afterCommitIfTransaction (t: Transaction, fn: Function) {
return fn()
}
-function deleteNonExistingModels <T extends { hasSameUniqueKeysThan (other: T): boolean } & Model<T>> (
+function deleteNonExistingModels <T extends { hasSameUniqueKeysThan (other: T): boolean } & Pick<Model, 'destroy'>> (
fromDatabase: T[],
newModels: T[],
t: Transaction
--
cgit v1.2.3
From c158a5faabb8ef0bc5d121fda4522d63603e8bc5 Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Wed, 12 May 2021 14:51:17 +0200
Subject: Refactor a little bit controllers
---
server/helpers/custom-validators/misc.ts | 2 +-
server/helpers/express-utils.ts | 28 ++++++++++++++--------------
2 files changed, 15 insertions(+), 15 deletions(-)
(limited to 'server/helpers')
diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts
index fd3b45804..229e9f03c 100644
--- a/server/helpers/custom-validators/misc.ts
+++ b/server/helpers/custom-validators/misc.ts
@@ -14,7 +14,7 @@ function isSafePath (p: string) {
})
}
-function isArray (value: any) {
+function isArray (value: any): value is any[] {
return Array.isArray(value)
}
diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts
index ede22a3cc..010c6961a 100644
--- a/server/helpers/express-utils.ts
+++ b/server/helpers/express-utils.ts
@@ -1,13 +1,13 @@
import * as express from 'express'
import * as multer from 'multer'
+import { extname } from 'path'
+import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes'
+import { CONFIG } from '../initializers/config'
import { REMOTE_SCHEME } from '../initializers/constants'
+import { isArray } from './custom-validators/misc'
import { logger } from './logger'
import { deleteFileAndCatch, generateRandomString } from './utils'
-import { extname } from 'path'
-import { isArray } from './custom-validators/misc'
-import { CONFIG } from '../initializers/config'
import { getExtFromMimetype } from './video'
-import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes'
function buildNSFWFilter (res?: express.Response, paramNSFW?: string) {
if (paramNSFW === 'true') return true
@@ -30,21 +30,21 @@ function buildNSFWFilter (res?: express.Response, paramNSFW?: string) {
return null
}
-function cleanUpReqFiles (req: { files: { [fieldname: string]: Express.Multer.File[] } | Express.Multer.File[] }) {
- const files = req.files
-
- if (!files) return
+function cleanUpReqFiles (
+ req: { files: { [fieldname: string]: Express.Multer.File[] } | Express.Multer.File[] }
+) {
+ const filesObject = req.files
+ if (!filesObject) return
- if (isArray(files)) {
- (files as Express.Multer.File[]).forEach(f => deleteFileAndCatch(f.path))
+ if (isArray(filesObject)) {
+ filesObject.forEach(f => deleteFileAndCatch(f.path))
return
}
- for (const key of Object.keys(files)) {
- const file = files[key]
+ for (const key of Object.keys(filesObject)) {
+ const files = filesObject[key]
- if (isArray(file)) file.forEach(f => deleteFileAndCatch(f.path))
- else deleteFileAndCatch(file.path)
+ files.forEach(f => deleteFileAndCatch(f.path))
}
}
--
cgit v1.2.3
From d61fa154af248c8456fb60da21507c767917d804 Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Tue, 25 May 2021 10:21:36 +0200
Subject: Fix duplicate ffmpeg preset option for live
---
server/helpers/ffmpeg-utils.ts | 1 -
1 file changed, 1 deletion(-)
(limited to 'server/helpers')
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts
index 25d9d4951..e328c49ac 100644
--- a/server/helpers/ffmpeg-utils.ts
+++ b/server/helpers/ffmpeg-utils.ts
@@ -236,7 +236,6 @@ async function getLiveTranscodingCommand (options: {
}
]
- command.outputOption('-preset superfast')
command.outputOption('-sc_threshold 0')
addDefaultEncoderGlobalParams({ command })
--
cgit v1.2.3
From 2539932e16129992a2c0889b4ff527c265a8e2c7 Mon Sep 17 00:00:00 2001
From: Chocobozzz <chocobozzz@cpy.re>
Date: Thu, 27 May 2021 15:59:55 +0200
Subject: Instance homepage support (#4007)
* Prepare homepage parsers
* Add ability to update instance hompage
* Add ability to set homepage as landing page
* Add homepage preview in admin
* Dynamically update left menu for homepage
* Inject home content in homepage
* Add videos list and channel miniature custom markup
* Remove unused elements in markup service
---
server/helpers/markdown.ts | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
(limited to 'server/helpers')
diff --git a/server/helpers/markdown.ts b/server/helpers/markdown.ts
index 2126bb752..41e57d857 100644
--- a/server/helpers/markdown.ts
+++ b/server/helpers/markdown.ts
@@ -1,4 +1,6 @@
-import { SANITIZE_OPTIONS, TEXT_WITH_HTML_RULES } from '@shared/core-utils'
+import { getSanitizeOptions, TEXT_WITH_HTML_RULES } from '@shared/core-utils'
+
+const sanitizeOptions = getSanitizeOptions()
const sanitizeHtml = require('sanitize-html')
const markdownItEmoji = require('markdown-it-emoji/light')
@@ -18,7 +20,7 @@ const toSafeHtml = text => {
const html = markdownIt.render(textWithLineFeed)
// Convert to safe Html
- return sanitizeHtml(html, SANITIZE_OPTIONS)
+ return sanitizeHtml(html, sanitizeOptions)
}
const mdToPlainText = text => {
@@ -28,7 +30,7 @@ const mdToPlainText = text => {
const html = markdownIt.render(text)
// Convert to safe Html
- const safeHtml = sanitizeHtml(html, SANITIZE_OPTIONS)
+ const safeHtml = sanitizeHtml(html, sanitizeOptions)
return safeHtml.replace(/<[^>]+>/g, '')
.replace(/\n$/, '')
--
cgit v1.2.3