aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/helpers
diff options
context:
space:
mode:
Diffstat (limited to 'server/helpers')
-rw-r--r--server/helpers/actor.ts16
-rw-r--r--server/helpers/audit-logger.ts2
-rw-r--r--server/helpers/core-utils.ts105
-rw-r--r--server/helpers/custom-validators/activitypub/playlist.ts7
-rw-r--r--server/helpers/custom-validators/misc.ts86
-rw-r--r--server/helpers/custom-validators/video-comments.ts81
-rw-r--r--server/helpers/custom-validators/video-imports.ts19
-rw-r--r--server/helpers/custom-validators/video-ownership.ts32
-rw-r--r--server/helpers/database-utils.ts31
-rw-r--r--server/helpers/express-utils.ts28
-rw-r--r--server/helpers/ffmpeg-utils.ts17
-rw-r--r--server/helpers/ffprobe-utils.ts3
-rw-r--r--server/helpers/image-utils.ts11
-rw-r--r--server/helpers/logger.ts5
-rw-r--r--server/helpers/markdown.ts8
-rw-r--r--server/helpers/middlewares/abuses.ts23
-rw-r--r--server/helpers/middlewares/accounts.ts65
-rw-r--r--server/helpers/middlewares/index.ts7
-rw-r--r--server/helpers/middlewares/video-blacklists.ts24
-rw-r--r--server/helpers/middlewares/video-captions.ts24
-rw-r--r--server/helpers/middlewares/video-channels.ts42
-rw-r--r--server/helpers/middlewares/video-playlists.ts39
-rw-r--r--server/helpers/middlewares/videos.ts125
-rw-r--r--server/helpers/promise-cache.ts21
-rw-r--r--server/helpers/requests.ts9
-rw-r--r--server/helpers/signup.ts62
-rw-r--r--server/helpers/uuid.ts32
-rw-r--r--server/helpers/video.ts67
-rw-r--r--server/helpers/webfinger.ts67
-rw-r--r--server/helpers/youtube-dl.ts553
30 files changed, 538 insertions, 1073 deletions
diff --git a/server/helpers/actor.ts b/server/helpers/actor.ts
deleted file mode 100644
index a60d3ed5d..000000000
--- a/server/helpers/actor.ts
+++ /dev/null
@@ -1,16 +0,0 @@
1
2import { ActorModel } from '../models/activitypub/actor'
3import { MActorAccountChannelId, MActorFull } from '../types/models'
4
5type ActorFetchByUrlType = 'all' | 'association-ids'
6
7function fetchActorByUrl (url: string, fetchType: ActorFetchByUrlType): Promise<MActorFull | MActorAccountChannelId> {
8 if (fetchType === 'all') return ActorModel.loadByUrlAndPopulateAccountAndChannel(url)
9
10 if (fetchType === 'association-ids') return ActorModel.loadByUrl(url)
11}
12
13export {
14 ActorFetchByUrlType,
15 fetchActorByUrl
16}
diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts
index 6aae5e821..884bd187d 100644
--- a/server/helpers/audit-logger.ts
+++ b/server/helpers/audit-logger.ts
@@ -7,7 +7,7 @@ import * as winston from 'winston'
7import { AUDIT_LOG_FILENAME } from '@server/initializers/constants' 7import { AUDIT_LOG_FILENAME } from '@server/initializers/constants'
8import { AdminAbuse, User, VideoChannel, VideoDetails, VideoImport } from '../../shared' 8import { AdminAbuse, User, VideoChannel, VideoDetails, VideoImport } from '../../shared'
9import { CustomConfig } from '../../shared/models/server/custom-config.model' 9import { CustomConfig } from '../../shared/models/server/custom-config.model'
10import { VideoComment } from '../../shared/models/videos/video-comment.model' 10import { VideoComment } from '../../shared/models/videos/comment/video-comment.model'
11import { CONFIG } from '../initializers/config' 11import { CONFIG } from '../initializers/config'
12import { jsonLoggerFormat, labelFormatter } from './logger' 12import { jsonLoggerFormat, labelFormatter } from './logger'
13 13
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index b93868c12..9abc532d2 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -8,7 +8,7 @@
8import { exec, ExecOptions } from 'child_process' 8import { exec, ExecOptions } from 'child_process'
9import { BinaryToTextEncoding, createHash, randomBytes } from 'crypto' 9import { BinaryToTextEncoding, createHash, randomBytes } from 'crypto'
10import { truncate } from 'lodash' 10import { truncate } from 'lodash'
11import { basename, isAbsolute, join, resolve } from 'path' 11import { basename, extname, isAbsolute, join, resolve } from 'path'
12import * as pem from 'pem' 12import * as pem from 'pem'
13import { pipeline } from 'stream' 13import { pipeline } from 'stream'
14import { URL } from 'url' 14import { URL } from 'url'
@@ -32,6 +32,18 @@ const objectConverter = (oldObject: any, keyConverter: (e: string) => string, va
32 return newObject 32 return newObject
33} 33}
34 34
35function mapToJSON (map: Map<any, any>) {
36 const obj: any = {}
37
38 for (const [ k, v ] of map) {
39 obj[k] = v
40 }
41
42 return obj
43}
44
45// ---------------------------------------------------------------------------
46
35const timeTable = { 47const timeTable = {
36 ms: 1, 48 ms: 1,
37 second: 1000, 49 second: 1000,
@@ -110,6 +122,8 @@ export function parseBytes (value: string | number): number {
110 } 122 }
111} 123}
112 124
125// ---------------------------------------------------------------------------
126
113function sanitizeUrl (url: string) { 127function sanitizeUrl (url: string) {
114 const urlObject = new URL(url) 128 const urlObject = new URL(url)
115 129
@@ -129,6 +143,8 @@ function sanitizeHost (host: string, remoteScheme: string) {
129 return host.replace(new RegExp(`:${toRemove}$`), '') 143 return host.replace(new RegExp(`:${toRemove}$`), '')
130} 144}
131 145
146// ---------------------------------------------------------------------------
147
132function isTestInstance () { 148function isTestInstance () {
133 return process.env.NODE_ENV === 'test' 149 return process.env.NODE_ENV === 'test'
134} 150}
@@ -141,6 +157,8 @@ function getAppNumber () {
141 return process.env.NODE_APP_INSTANCE 157 return process.env.NODE_APP_INSTANCE
142} 158}
143 159
160// ---------------------------------------------------------------------------
161
144let rootPath: string 162let rootPath: string
145 163
146function root () { 164function root () {
@@ -154,27 +172,19 @@ function root () {
154 return rootPath 172 return rootPath
155} 173}
156 174
157function pageToStartAndCount (page: number, itemsPerPage: number) { 175function buildPath (path: string) {
158 const start = (page - 1) * itemsPerPage 176 if (isAbsolute(path)) return path
159 177
160 return { start, count: itemsPerPage } 178 return join(root(), path)
161} 179}
162 180
163function mapToJSON (map: Map<any, any>) { 181function getLowercaseExtension (filename: string) {
164 const obj: any = {} 182 const ext = extname(filename) || ''
165 183
166 for (const [ k, v ] of map) { 184 return ext.toLowerCase()
167 obj[k] = v
168 }
169
170 return obj
171} 185}
172 186
173function buildPath (path: string) { 187// ---------------------------------------------------------------------------
174 if (isAbsolute(path)) return path
175
176 return join(root(), path)
177}
178 188
179// Consistent with .length, lodash truncate function is not 189// Consistent with .length, lodash truncate function is not
180function peertubeTruncate (str: string, options: { length: number, separator?: RegExp, omission?: string }) { 190function peertubeTruncate (str: string, options: { length: number, separator?: RegExp, omission?: string }) {
@@ -189,6 +199,27 @@ function peertubeTruncate (str: string, options: { length: number, separator?: R
189 return truncate(str, options) 199 return truncate(str, options)
190} 200}
191 201
202function pageToStartAndCount (page: number, itemsPerPage: number) {
203 const start = (page - 1) * itemsPerPage
204
205 return { start, count: itemsPerPage }
206}
207
208// ---------------------------------------------------------------------------
209
210type SemVersion = { major: number, minor: number, patch: number }
211function parseSemVersion (s: string) {
212 const parsed = s.match(/^v?(\d+)\.(\d+)\.(\d+)$/i)
213
214 return {
215 major: parseInt(parsed[1]),
216 minor: parseInt(parsed[2]),
217 patch: parseInt(parsed[3])
218 } as SemVersion
219}
220
221// ---------------------------------------------------------------------------
222
192function sha256 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') { 223function sha256 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') {
193 return createHash('sha256').update(str).digest(encoding) 224 return createHash('sha256').update(str).digest(encoding)
194} 225}
@@ -197,6 +228,8 @@ function sha1 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') {
197 return createHash('sha1').update(str).digest(encoding) 228 return createHash('sha1').update(str).digest(encoding)
198} 229}
199 230
231// ---------------------------------------------------------------------------
232
200function execShell (command: string, options?: ExecOptions) { 233function execShell (command: string, options?: ExecOptions) {
201 return new Promise<{ err?: Error, stdout: string, stderr: string }>((res, rej) => { 234 return new Promise<{ err?: Error, stdout: string, stderr: string }>((res, rej) => {
202 exec(command, options, (err, stdout, stderr) => { 235 exec(command, options, (err, stdout, stderr) => {
@@ -208,6 +241,20 @@ function execShell (command: string, options?: ExecOptions) {
208 }) 241 })
209} 242}
210 243
244// ---------------------------------------------------------------------------
245
246function isOdd (num: number) {
247 return (num % 2) !== 0
248}
249
250function toEven (num: number) {
251 if (isOdd(num)) return num + 1
252
253 return num
254}
255
256// ---------------------------------------------------------------------------
257
211function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> { 258function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> {
212 return function promisified (): Promise<A> { 259 return function promisified (): Promise<A> {
213 return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { 260 return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
@@ -233,17 +280,6 @@ function promisify2<T, U, A> (func: (arg1: T, arg2: U, cb: (err: any, result: A)
233 } 280 }
234} 281}
235 282
236type SemVersion = { major: number, minor: number, patch: number }
237function parseSemVersion (s: string) {
238 const parsed = s.match(/^v?(\d+)\.(\d+)\.(\d+)$/i)
239
240 return {
241 major: parseInt(parsed[1]),
242 minor: parseInt(parsed[2]),
243 patch: parseInt(parsed[3])
244 } as SemVersion
245}
246
247const randomBytesPromise = promisify1<number, Buffer>(randomBytes) 283const randomBytesPromise = promisify1<number, Buffer>(randomBytes)
248const createPrivateKey = promisify1<number, { key: string }>(pem.createPrivateKey) 284const createPrivateKey = promisify1<number, { key: string }>(pem.createPrivateKey)
249const getPublicKey = promisify1<string, { publicKey: string }>(pem.getPublicKey) 285const getPublicKey = promisify1<string, { publicKey: string }>(pem.getPublicKey)
@@ -259,17 +295,21 @@ export {
259 getAppNumber, 295 getAppNumber,
260 296
261 objectConverter, 297 objectConverter,
298 mapToJSON,
299
262 root, 300 root,
263 pageToStartAndCount, 301 buildPath,
302 getLowercaseExtension,
264 sanitizeUrl, 303 sanitizeUrl,
265 sanitizeHost, 304 sanitizeHost,
266 buildPath, 305
267 execShell, 306 execShell,
307
308 pageToStartAndCount,
268 peertubeTruncate, 309 peertubeTruncate,
269 310
270 sha256, 311 sha256,
271 sha1, 312 sha1,
272 mapToJSON,
273 313
274 promisify0, 314 promisify0,
275 promisify1, 315 promisify1,
@@ -282,5 +322,8 @@ export {
282 execPromise, 322 execPromise,
283 pipelinePromise, 323 pipelinePromise,
284 324
285 parseSemVersion 325 parseSemVersion,
326
327 isOdd,
328 toEven
286} 329}
diff --git a/server/helpers/custom-validators/activitypub/playlist.ts b/server/helpers/custom-validators/activitypub/playlist.ts
index bd0d16a4a..72c5b80e9 100644
--- a/server/helpers/custom-validators/activitypub/playlist.ts
+++ b/server/helpers/custom-validators/activitypub/playlist.ts
@@ -1,13 +1,16 @@
1import { exists, isDateValid } from '../misc'
2import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object'
3import validator from 'validator' 1import validator from 'validator'
4import { PlaylistElementObject } from '../../../../shared/models/activitypub/objects/playlist-element-object' 2import { PlaylistElementObject } from '../../../../shared/models/activitypub/objects/playlist-element-object'
3import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object'
4import { exists, isDateValid, isUUIDValid } from '../misc'
5import { isVideoPlaylistNameValid } from '../video-playlists'
5import { isActivityPubUrlValid } from './misc' 6import { isActivityPubUrlValid } from './misc'
6 7
7function isPlaylistObjectValid (object: PlaylistObject) { 8function isPlaylistObjectValid (object: PlaylistObject) {
8 return exists(object) && 9 return exists(object) &&
9 object.type === 'Playlist' && 10 object.type === 'Playlist' &&
10 validator.isInt(object.totalItems + '') && 11 validator.isInt(object.totalItems + '') &&
12 isVideoPlaylistNameValid(object.name) &&
13 isUUIDValid(object.uuid) &&
11 isDateValid(object.published) && 14 isDateValid(object.published) &&
12 isDateValid(object.updated) 15 isDateValid(object.updated)
13} 16}
diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts
index fd3b45804..528bfcfb8 100644
--- a/server/helpers/custom-validators/misc.ts
+++ b/server/helpers/custom-validators/misc.ts
@@ -2,6 +2,7 @@ import 'multer'
2import { UploadFilesForCheck } from 'express' 2import { UploadFilesForCheck } from 'express'
3import { sep } from 'path' 3import { sep } from 'path'
4import validator from 'validator' 4import validator from 'validator'
5import { isShortUUID, shortToUUID } from '../uuid'
5 6
6function exists (value: any) { 7function exists (value: any) {
7 return value !== undefined && value !== null 8 return value !== undefined && value !== null
@@ -14,7 +15,7 @@ function isSafePath (p: string) {
14 }) 15 })
15} 16}
16 17
17function isArray (value: any) { 18function isArray (value: any): value is any[] {
18 return Array.isArray(value) 19 return Array.isArray(value)
19} 20}
20 21
@@ -50,42 +51,7 @@ function isIntOrNull (value: any) {
50 return value === null || validator.isInt('' + value) 51 return value === null || validator.isInt('' + value)
51} 52}
52 53
53function toIntOrNull (value: string) { 54// ---------------------------------------------------------------------------
54 const v = toValueOrNull(value)
55
56 if (v === null || v === undefined) return v
57 if (typeof v === 'number') return v
58
59 return validator.toInt('' + v)
60}
61
62function toBooleanOrNull (value: any) {
63 const v = toValueOrNull(value)
64
65 if (v === null || v === undefined) return v
66 if (typeof v === 'boolean') return v
67
68 return validator.toBoolean('' + v)
69}
70
71function toValueOrNull (value: string) {
72 if (value === 'null') return null
73
74 return value
75}
76
77function toArray (value: any) {
78 if (value && isArray(value) === false) return [ value ]
79
80 return value
81}
82
83function toIntArray (value: any) {
84 if (!value) return []
85 if (isArray(value) === false) return [ validator.toInt(value) ]
86
87 return value.map(v => validator.toInt(v))
88}
89 55
90function isFileFieldValid ( 56function isFileFieldValid (
91 files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[], 57 files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[],
@@ -160,6 +126,51 @@ function isFileValid (
160 126
161// --------------------------------------------------------------------------- 127// ---------------------------------------------------------------------------
162 128
129function toCompleteUUID (value: string) {
130 if (isShortUUID(value)) return shortToUUID(value)
131
132 return value
133}
134
135function toIntOrNull (value: string) {
136 const v = toValueOrNull(value)
137
138 if (v === null || v === undefined) return v
139 if (typeof v === 'number') return v
140
141 return validator.toInt('' + v)
142}
143
144function toBooleanOrNull (value: any) {
145 const v = toValueOrNull(value)
146
147 if (v === null || v === undefined) return v
148 if (typeof v === 'boolean') return v
149
150 return validator.toBoolean('' + v)
151}
152
153function toValueOrNull (value: string) {
154 if (value === 'null') return null
155
156 return value
157}
158
159function toArray (value: any) {
160 if (value && isArray(value) === false) return [ value ]
161
162 return value
163}
164
165function toIntArray (value: any) {
166 if (!value) return []
167 if (isArray(value) === false) return [ validator.toInt(value) ]
168
169 return value.map(v => validator.toInt(v))
170}
171
172// ---------------------------------------------------------------------------
173
163export { 174export {
164 exists, 175 exists,
165 isArrayOf, 176 isArrayOf,
@@ -169,6 +180,7 @@ export {
169 isIdValid, 180 isIdValid,
170 isSafePath, 181 isSafePath,
171 isUUIDValid, 182 isUUIDValid,
183 toCompleteUUID,
172 isIdOrUUIDValid, 184 isIdOrUUIDValid,
173 isDateValid, 185 isDateValid,
174 toValueOrNull, 186 toValueOrNull,
diff --git a/server/helpers/custom-validators/video-comments.ts b/server/helpers/custom-validators/video-comments.ts
index 8d3ce580e..94bdf237a 100644
--- a/server/helpers/custom-validators/video-comments.ts
+++ b/server/helpers/custom-validators/video-comments.ts
@@ -1,9 +1,5 @@
1import * as express from 'express'
2import validator from 'validator' 1import validator from 'validator'
3import { VideoCommentModel } from '@server/models/video/video-comment'
4import { CONSTRAINTS_FIELDS } from '../../initializers/constants' 2import { CONSTRAINTS_FIELDS } from '../../initializers/constants'
5import { MVideoId } from '@server/types/models'
6import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
7 3
8const VIDEO_COMMENTS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_COMMENTS 4const VIDEO_COMMENTS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_COMMENTS
9 5
@@ -11,83 +7,8 @@ function isValidVideoCommentText (value: string) {
11 return value === null || validator.isLength(value, VIDEO_COMMENTS_CONSTRAINTS_FIELDS.TEXT) 7 return value === null || validator.isLength(value, VIDEO_COMMENTS_CONSTRAINTS_FIELDS.TEXT)
12} 8}
13 9
14async function doesVideoCommentThreadExist (idArg: number | string, video: MVideoId, res: express.Response) {
15 const id = parseInt(idArg + '', 10)
16 const videoComment = await VideoCommentModel.loadById(id)
17
18 if (!videoComment) {
19 res.status(HttpStatusCode.NOT_FOUND_404)
20 .json({ error: 'Video comment thread not found' })
21 .end()
22
23 return false
24 }
25
26 if (videoComment.videoId !== video.id) {
27 res.status(HttpStatusCode.BAD_REQUEST_400)
28 .json({ error: 'Video comment is not associated to this video.' })
29 .end()
30
31 return false
32 }
33
34 if (videoComment.inReplyToCommentId !== null) {
35 res.status(HttpStatusCode.BAD_REQUEST_400)
36 .json({ error: 'Video comment is not a thread.' })
37 .end()
38
39 return false
40 }
41
42 res.locals.videoCommentThread = videoComment
43 return true
44}
45
46async function doesVideoCommentExist (idArg: number | string, video: MVideoId, res: express.Response) {
47 const id = parseInt(idArg + '', 10)
48 const videoComment = await VideoCommentModel.loadByIdAndPopulateVideoAndAccountAndReply(id)
49
50 if (!videoComment) {
51 res.status(HttpStatusCode.NOT_FOUND_404)
52 .json({ error: 'Video comment thread not found' })
53 .end()
54
55 return false
56 }
57
58 if (videoComment.videoId !== video.id) {
59 res.status(HttpStatusCode.BAD_REQUEST_400)
60 .json({ error: 'Video comment is not associated to this video.' })
61 .end()
62
63 return false
64 }
65
66 res.locals.videoCommentFull = videoComment
67 return true
68}
69
70async function doesCommentIdExist (idArg: number | string, res: express.Response) {
71 const id = parseInt(idArg + '', 10)
72 const videoComment = await VideoCommentModel.loadByIdAndPopulateVideoAndAccountAndReply(id)
73
74 if (!videoComment) {
75 res.status(HttpStatusCode.NOT_FOUND_404)
76 .json({ error: 'Video comment thread not found' })
77
78 return false
79 }
80
81 res.locals.videoCommentFull = videoComment
82
83 return true
84}
85
86// --------------------------------------------------------------------------- 10// ---------------------------------------------------------------------------
87 11
88export { 12export {
89 isValidVideoCommentText, 13 isValidVideoCommentText
90 doesVideoCommentThreadExist,
91 doesVideoCommentExist,
92 doesCommentIdExist
93} 14}
diff --git a/server/helpers/custom-validators/video-imports.ts b/server/helpers/custom-validators/video-imports.ts
index 0063d3337..dbf6a3504 100644
--- a/server/helpers/custom-validators/video-imports.ts
+++ b/server/helpers/custom-validators/video-imports.ts
@@ -2,9 +2,6 @@ import 'multer'
2import validator from 'validator' 2import validator from 'validator'
3import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_IMPORT_STATES } from '../../initializers/constants' 3import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_IMPORT_STATES } from '../../initializers/constants'
4import { exists, isFileValid } from './misc' 4import { exists, isFileValid } from './misc'
5import * as express from 'express'
6import { VideoImportModel } from '../../models/video/video-import'
7import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
8 5
9function isVideoImportTargetUrlValid (url: string) { 6function isVideoImportTargetUrlValid (url: string) {
10 const isURLOptions = { 7 const isURLOptions = {
@@ -32,26 +29,10 @@ function isVideoImportTorrentFile (files: { [ fieldname: string ]: Express.Multe
32 return isFileValid(files, videoTorrentImportRegex, 'torrentfile', CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_FILE.FILE_SIZE.max, true) 29 return isFileValid(files, videoTorrentImportRegex, 'torrentfile', CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_FILE.FILE_SIZE.max, true)
33} 30}
34 31
35async function doesVideoImportExist (id: number, res: express.Response) {
36 const videoImport = await VideoImportModel.loadAndPopulateVideo(id)
37
38 if (!videoImport) {
39 res.status(HttpStatusCode.NOT_FOUND_404)
40 .json({ error: 'Video import not found' })
41 .end()
42
43 return false
44 }
45
46 res.locals.videoImport = videoImport
47 return true
48}
49
50// --------------------------------------------------------------------------- 32// ---------------------------------------------------------------------------
51 33
52export { 34export {
53 isVideoImportStateValid, 35 isVideoImportStateValid,
54 isVideoImportTargetUrlValid, 36 isVideoImportTargetUrlValid,
55 doesVideoImportExist,
56 isVideoImportTorrentFile 37 isVideoImportTorrentFile
57} 38}
diff --git a/server/helpers/custom-validators/video-ownership.ts b/server/helpers/custom-validators/video-ownership.ts
index ee3cebe10..0e1c63bad 100644
--- a/server/helpers/custom-validators/video-ownership.ts
+++ b/server/helpers/custom-validators/video-ownership.ts
@@ -1,32 +1,20 @@
1import { Response } from 'express' 1import { Response } from 'express'
2import { VideoChangeOwnershipModel } from '../../models/video/video-change-ownership'
3import { MVideoChangeOwnershipFull } from '@server/types/models/video/video-change-ownership'
4import { MUserId } from '@server/types/models' 2import { MUserId } from '@server/types/models'
3import { MVideoChangeOwnershipFull } from '@server/types/models/video/video-change-ownership'
5import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' 4import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
6 5
7export async function doesChangeVideoOwnershipExist (idArg: number | string, res: Response) { 6function checkUserCanTerminateOwnershipChange (user: MUserId, videoChangeOwnership: MVideoChangeOwnershipFull, res: Response) {
8 const id = parseInt(idArg + '', 10)
9 const videoChangeOwnership = await VideoChangeOwnershipModel.load(id)
10
11 if (!videoChangeOwnership) {
12 res.status(HttpStatusCode.NOT_FOUND_404)
13 .json({ error: 'Video change ownership not found' })
14 .end()
15
16 return false
17 }
18
19 res.locals.videoChangeOwnership = videoChangeOwnership
20 return true
21}
22
23export function checkUserCanTerminateOwnershipChange (user: MUserId, videoChangeOwnership: MVideoChangeOwnershipFull, res: Response) {
24 if (videoChangeOwnership.NextOwner.userId === user.id) { 7 if (videoChangeOwnership.NextOwner.userId === user.id) {
25 return true 8 return true
26 } 9 }
27 10
28 res.status(HttpStatusCode.FORBIDDEN_403) 11 res.fail({
29 .json({ error: 'Cannot terminate an ownership change of another user' }) 12 status: HttpStatusCode.FORBIDDEN_403,
30 .end() 13 message: 'Cannot terminate an ownership change of another user'
14 })
31 return false 15 return false
32} 16}
17
18export {
19 checkUserCanTerminateOwnershipChange
20}
diff --git a/server/helpers/database-utils.ts b/server/helpers/database-utils.ts
index f9cb33aca..b5dc70c17 100644
--- a/server/helpers/database-utils.ts
+++ b/server/helpers/database-utils.ts
@@ -58,7 +58,7 @@ function transactionRetryer <T> (func: (err: any, data: T) => any) {
58 58
59 errorFilter: err => { 59 errorFilter: err => {
60 const willRetry = (err.name === 'SequelizeDatabaseError') 60 const willRetry = (err.name === 'SequelizeDatabaseError')
61 logger.debug('Maybe retrying the transaction function.', { willRetry, err }) 61 logger.debug('Maybe retrying the transaction function.', { willRetry, err, tags: [ 'sql', 'retry' ] })
62 return willRetry 62 return willRetry
63 } 63 }
64 }, 64 },
@@ -68,7 +68,9 @@ function transactionRetryer <T> (func: (err: any, data: T) => any) {
68 }) 68 })
69} 69}
70 70
71function updateInstanceWithAnother <T extends Model<T>> (instanceToUpdate: Model<T>, baseInstance: Model<T>) { 71// ---------------------------------------------------------------------------
72
73function updateInstanceWithAnother <M, T extends U, U extends Model<M>> (instanceToUpdate: T, baseInstance: U) {
72 const obj = baseInstance.toJSON() 74 const obj = baseInstance.toJSON()
73 75
74 for (const key of Object.keys(obj)) { 76 for (const key of Object.keys(obj)) {
@@ -82,13 +84,7 @@ function resetSequelizeInstance (instance: Model<any>, savedFields: object) {
82 }) 84 })
83} 85}
84 86
85function afterCommitIfTransaction (t: Transaction, fn: Function) { 87function deleteNonExistingModels <T extends { hasSameUniqueKeysThan (other: T): boolean } & Pick<Model, 'destroy'>> (
86 if (t) return t.afterCommit(() => fn())
87
88 return fn()
89}
90
91function deleteNonExistingModels <T extends { hasSameUniqueKeysThan (other: T): boolean } & Model<T>> (
92 fromDatabase: T[], 88 fromDatabase: T[],
93 newModels: T[], 89 newModels: T[],
94 t: Transaction 90 t: Transaction
@@ -111,6 +107,20 @@ function setAsUpdated (table: string, id: number, transaction?: Transaction) {
111 107
112// --------------------------------------------------------------------------- 108// ---------------------------------------------------------------------------
113 109
110function runInReadCommittedTransaction <T> (fn: (t: Transaction) => Promise<T>) {
111 const options = { isolationLevel: Transaction.ISOLATION_LEVELS.READ_COMMITTED }
112
113 return sequelizeTypescript.transaction(options, t => fn(t))
114}
115
116function afterCommitIfTransaction (t: Transaction, fn: Function) {
117 if (t) return t.afterCommit(() => fn())
118
119 return fn()
120}
121
122// ---------------------------------------------------------------------------
123
114export { 124export {
115 resetSequelizeInstance, 125 resetSequelizeInstance,
116 retryTransactionWrapper, 126 retryTransactionWrapper,
@@ -118,5 +128,6 @@ export {
118 updateInstanceWithAnother, 128 updateInstanceWithAnother,
119 afterCommitIfTransaction, 129 afterCommitIfTransaction,
120 deleteNonExistingModels, 130 deleteNonExistingModels,
121 setAsUpdated 131 setAsUpdated,
132 runInReadCommittedTransaction
122} 133}
diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts
index ede22a3cc..0ff113274 100644
--- a/server/helpers/express-utils.ts
+++ b/server/helpers/express-utils.ts
@@ -1,13 +1,13 @@
1import * as express from 'express' 1import * as express from 'express'
2import * as multer from 'multer' 2import * as multer from 'multer'
3import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes'
4import { CONFIG } from '../initializers/config'
3import { REMOTE_SCHEME } from '../initializers/constants' 5import { REMOTE_SCHEME } from '../initializers/constants'
6import { getLowercaseExtension } from './core-utils'
7import { isArray } from './custom-validators/misc'
4import { logger } from './logger' 8import { logger } from './logger'
5import { deleteFileAndCatch, generateRandomString } from './utils' 9import { deleteFileAndCatch, generateRandomString } from './utils'
6import { extname } from 'path'
7import { isArray } from './custom-validators/misc'
8import { CONFIG } from '../initializers/config'
9import { getExtFromMimetype } from './video' 10import { getExtFromMimetype } from './video'
10import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes'
11 11
12function buildNSFWFilter (res?: express.Response, paramNSFW?: string) { 12function buildNSFWFilter (res?: express.Response, paramNSFW?: string) {
13 if (paramNSFW === 'true') return true 13 if (paramNSFW === 'true') return true
@@ -30,21 +30,19 @@ function buildNSFWFilter (res?: express.Response, paramNSFW?: string) {
30 return null 30 return null
31} 31}
32 32
33function cleanUpReqFiles (req: { files: { [fieldname: string]: Express.Multer.File[] } | Express.Multer.File[] }) { 33function cleanUpReqFiles (req: express.Request) {
34 const files = req.files 34 const filesObject = req.files
35 35 if (!filesObject) return
36 if (!files) return
37 36
38 if (isArray(files)) { 37 if (isArray(filesObject)) {
39 (files as Express.Multer.File[]).forEach(f => deleteFileAndCatch(f.path)) 38 filesObject.forEach(f => deleteFileAndCatch(f.path))
40 return 39 return
41 } 40 }
42 41
43 for (const key of Object.keys(files)) { 42 for (const key of Object.keys(filesObject)) {
44 const file = files[key] 43 const files = filesObject[key]
45 44
46 if (isArray(file)) file.forEach(f => deleteFileAndCatch(f.path)) 45 files.forEach(f => deleteFileAndCatch(f.path))
47 else deleteFileAndCatch(file.path)
48 } 46 }
49} 47}
50 48
@@ -79,7 +77,7 @@ function createReqFiles (
79 77
80 filename: async (req, file, cb) => { 78 filename: async (req, file, cb) => {
81 let extension: string 79 let extension: string
82 const fileExtension = extname(file.originalname) 80 const fileExtension = getLowercaseExtension(file.originalname)
83 const extensionFromMimetype = getExtFromMimetype(mimeTypes, file.mimetype) 81 const extensionFromMimetype = getExtFromMimetype(mimeTypes, file.mimetype)
84 82
85 // Take the file extension if we don't understand the mime type 83 // Take the file extension if we don't understand the mime type
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts
index e328c49ac..6f5a71b4a 100644
--- a/server/helpers/ffmpeg-utils.ts
+++ b/server/helpers/ffmpeg-utils.ts
@@ -3,13 +3,12 @@ import * as ffmpeg from 'fluent-ffmpeg'
3import { readFile, remove, writeFile } from 'fs-extra' 3import { readFile, remove, writeFile } from 'fs-extra'
4import { dirname, join } from 'path' 4import { dirname, join } from 'path'
5import { FFMPEG_NICE, VIDEO_LIVE } from '@server/initializers/constants' 5import { FFMPEG_NICE, VIDEO_LIVE } from '@server/initializers/constants'
6import { AvailableEncoders, EncoderOptionsBuilder, EncoderOptions, EncoderProfile, VideoResolution } from '../../shared/models/videos' 6import { AvailableEncoders, EncoderOptions, EncoderOptionsBuilder, EncoderProfile, VideoResolution } from '../../shared/models/videos'
7import { CONFIG } from '../initializers/config' 7import { CONFIG } from '../initializers/config'
8import { execPromise, promisify0 } from './core-utils' 8import { execPromise, promisify0 } from './core-utils'
9import { computeFPS, getAudioStream, getVideoFileFPS } from './ffprobe-utils' 9import { computeFPS, getAudioStream, getVideoFileFPS } from './ffprobe-utils'
10import { processImage } from './image-utils' 10import { processImage } from './image-utils'
11import { logger } from './logger' 11import { logger } from './logger'
12import { FilterSpecification } from 'fluent-ffmpeg'
13 12
14/** 13/**
15 * 14 *
@@ -133,7 +132,7 @@ interface BaseTranscodeOptions {
133 availableEncoders: AvailableEncoders 132 availableEncoders: AvailableEncoders
134 profile: string 133 profile: string
135 134
136 resolution: VideoResolution 135 resolution: number
137 136
138 isPortraitMode?: boolean 137 isPortraitMode?: boolean
139 138
@@ -227,7 +226,7 @@ async function getLiveTranscodingCommand (options: {
227 226
228 const varStreamMap: string[] = [] 227 const varStreamMap: string[] = []
229 228
230 const complexFilter: FilterSpecification[] = [ 229 const complexFilter: ffmpeg.FilterSpecification[] = [
231 { 230 {
232 inputs: '[v:0]', 231 inputs: '[v:0]',
233 filter: 'split', 232 filter: 'split',
@@ -407,8 +406,7 @@ async function buildx264VODCommand (command: ffmpeg.FfmpegCommand, options: Tran
407async function buildAudioMergeCommand (command: ffmpeg.FfmpegCommand, options: MergeAudioTranscodeOptions) { 406async function buildAudioMergeCommand (command: ffmpeg.FfmpegCommand, options: MergeAudioTranscodeOptions) {
408 command = command.loop(undefined) 407 command = command.loop(undefined)
409 408
410 // Avoid "height not divisible by 2" error 409 const scaleFilterValue = getScaleCleanerValue()
411 const scaleFilterValue = 'trunc(iw/2)*2:trunc(ih/2)*2'
412 command = await presetVideo({ command, input: options.audioPath, transcodeOptions: options, scaleFilterValue }) 410 command = await presetVideo({ command, input: options.audioPath, transcodeOptions: options, scaleFilterValue })
413 411
414 command.outputOption('-preset:v veryfast') 412 command.outputOption('-preset:v veryfast')
@@ -542,7 +540,7 @@ async function getEncoderBuilderResult (options: {
542 } 540 }
543 } 541 }
544 542
545 const result = await builder({ input, resolution: resolution, fps, streamNum }) 543 const result = await builder({ input, resolution, fps, streamNum })
546 544
547 return { 545 return {
548 result, 546 result,
@@ -727,6 +725,11 @@ async function runCommand (options: {
727 }) 725 })
728} 726}
729 727
728// Avoid "height not divisible by 2" error
729function getScaleCleanerValue () {
730 return 'trunc(iw/2)*2:trunc(ih/2)*2'
731}
732
730// --------------------------------------------------------------------------- 733// ---------------------------------------------------------------------------
731 734
732export { 735export {
diff --git a/server/helpers/ffprobe-utils.ts b/server/helpers/ffprobe-utils.ts
index 40eaafd57..ef2aa3f89 100644
--- a/server/helpers/ffprobe-utils.ts
+++ b/server/helpers/ffprobe-utils.ts
@@ -1,6 +1,5 @@
1import * as ffmpeg from 'fluent-ffmpeg' 1import * as ffmpeg from 'fluent-ffmpeg'
2import { VideoFileMetadata } from '@shared/models/videos/video-file-metadata' 2import { getMaxBitrate, VideoFileMetadata, VideoResolution } from '../../shared/models/videos'
3import { getMaxBitrate, VideoResolution } from '../../shared/models/videos'
4import { CONFIG } from '../initializers/config' 3import { CONFIG } from '../initializers/config'
5import { VIDEO_TRANSCODING_FPS } from '../initializers/constants' 4import { VIDEO_TRANSCODING_FPS } from '../initializers/constants'
6import { logger } from './logger' 5import { logger } from './logger'
diff --git a/server/helpers/image-utils.ts b/server/helpers/image-utils.ts
index 6f6f8d4da..c76ed545b 100644
--- a/server/helpers/image-utils.ts
+++ b/server/helpers/image-utils.ts
@@ -1,12 +1,12 @@
1import { copy, readFile, remove, rename } from 'fs-extra' 1import { copy, readFile, remove, rename } from 'fs-extra'
2import * as Jimp from 'jimp' 2import * as Jimp from 'jimp'
3import { extname } from 'path' 3import { getLowercaseExtension } from './core-utils'
4import { v4 as uuidv4 } from 'uuid'
5import { convertWebPToJPG, processGIF } from './ffmpeg-utils' 4import { convertWebPToJPG, processGIF } from './ffmpeg-utils'
6import { logger } from './logger' 5import { logger } from './logger'
6import { buildUUID } from './uuid'
7 7
8function generateImageFilename (extension = '.jpg') { 8function generateImageFilename (extension = '.jpg') {
9 return uuidv4() + extension 9 return buildUUID() + extension
10} 10}
11 11
12async function processImage ( 12async function processImage (
@@ -15,7 +15,7 @@ async function processImage (
15 newSize: { width: number, height: number }, 15 newSize: { width: number, height: number },
16 keepOriginal = false 16 keepOriginal = false
17) { 17) {
18 const extension = extname(path) 18 const extension = getLowercaseExtension(path)
19 19
20 if (path === destination) { 20 if (path === destination) {
21 throw new Error('Jimp/FFmpeg needs an input path different that the output path.') 21 throw new Error('Jimp/FFmpeg needs an input path different that the output path.')
@@ -61,7 +61,8 @@ async function jimpProcessor (path: string, destination: string, newSize: { widt
61 await remove(destination) 61 await remove(destination)
62 62
63 // Optimization if the source file has the appropriate size 63 // Optimization if the source file has the appropriate size
64 if (await skipProcessing({ jimpInstance, newSize, imageBytes: inputBuffer.byteLength, inputExt, outputExt: extname(destination) })) { 64 const outputExt = getLowercaseExtension(destination)
65 if (skipProcessing({ jimpInstance, newSize, imageBytes: inputBuffer.byteLength, inputExt, outputExt })) {
65 return copy(path, destination) 66 return copy(path, destination)
66 } 67 }
67 68
diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts
index a112fd300..29e06860d 100644
--- a/server/helpers/logger.ts
+++ b/server/helpers/logger.ts
@@ -151,7 +151,8 @@ const bunyanLogger = {
151 fatal: bunyanLogFactory('error') 151 fatal: bunyanLogFactory('error')
152} 152}
153 153
154function loggerTagsFactory (...defaultTags: string[]) { 154type LoggerTagsFn = (...tags: string[]) => { tags: string[] }
155function loggerTagsFactory (...defaultTags: string[]): LoggerTagsFn {
155 return (...tags: string[]) => { 156 return (...tags: string[]) => {
156 return { tags: defaultTags.concat(tags) } 157 return { tags: defaultTags.concat(tags) }
157 } 158 }
@@ -160,6 +161,8 @@ function loggerTagsFactory (...defaultTags: string[]) {
160// --------------------------------------------------------------------------- 161// ---------------------------------------------------------------------------
161 162
162export { 163export {
164 LoggerTagsFn,
165
163 buildLogger, 166 buildLogger,
164 timestampFormatter, 167 timestampFormatter,
165 labelFormatter, 168 labelFormatter,
diff --git a/server/helpers/markdown.ts b/server/helpers/markdown.ts
index 2126bb752..41e57d857 100644
--- a/server/helpers/markdown.ts
+++ b/server/helpers/markdown.ts
@@ -1,4 +1,6 @@
1import { SANITIZE_OPTIONS, TEXT_WITH_HTML_RULES } from '@shared/core-utils' 1import { getSanitizeOptions, TEXT_WITH_HTML_RULES } from '@shared/core-utils'
2
3const sanitizeOptions = getSanitizeOptions()
2 4
3const sanitizeHtml = require('sanitize-html') 5const sanitizeHtml = require('sanitize-html')
4const markdownItEmoji = require('markdown-it-emoji/light') 6const markdownItEmoji = require('markdown-it-emoji/light')
@@ -18,7 +20,7 @@ const toSafeHtml = text => {
18 const html = markdownIt.render(textWithLineFeed) 20 const html = markdownIt.render(textWithLineFeed)
19 21
20 // Convert to safe Html 22 // Convert to safe Html
21 return sanitizeHtml(html, SANITIZE_OPTIONS) 23 return sanitizeHtml(html, sanitizeOptions)
22} 24}
23 25
24const mdToPlainText = text => { 26const mdToPlainText = text => {
@@ -28,7 +30,7 @@ const mdToPlainText = text => {
28 const html = markdownIt.render(text) 30 const html = markdownIt.render(text)
29 31
30 // Convert to safe Html 32 // Convert to safe Html
31 const safeHtml = sanitizeHtml(html, SANITIZE_OPTIONS) 33 const safeHtml = sanitizeHtml(html, sanitizeOptions)
32 34
33 return safeHtml.replace(/<[^>]+>/g, '') 35 return safeHtml.replace(/<[^>]+>/g, '')
34 .replace(/\n$/, '') 36 .replace(/\n$/, '')
diff --git a/server/helpers/middlewares/abuses.ts b/server/helpers/middlewares/abuses.ts
deleted file mode 100644
index c53bd9efd..000000000
--- a/server/helpers/middlewares/abuses.ts
+++ /dev/null
@@ -1,23 +0,0 @@
1import { Response } from 'express'
2import { AbuseModel } from '../../models/abuse/abuse'
3import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
4
5async function doesAbuseExist (abuseId: number | string, res: Response) {
6 const abuse = await AbuseModel.loadByIdWithReporter(parseInt(abuseId + '', 10))
7
8 if (!abuse) {
9 res.status(HttpStatusCode.NOT_FOUND_404)
10 .json({ error: 'Abuse not found' })
11
12 return false
13 }
14
15 res.locals.abuse = abuse
16 return true
17}
18
19// ---------------------------------------------------------------------------
20
21export {
22 doesAbuseExist
23}
diff --git a/server/helpers/middlewares/accounts.ts b/server/helpers/middlewares/accounts.ts
deleted file mode 100644
index 13ae6cdf4..000000000
--- a/server/helpers/middlewares/accounts.ts
+++ /dev/null
@@ -1,65 +0,0 @@
1import { Response } from 'express'
2import { UserModel } from '@server/models/account/user'
3import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
4import { AccountModel } from '../../models/account/account'
5import { MAccountDefault } from '../../types/models'
6
7function doesAccountIdExist (id: number | string, res: Response, sendNotFound = true) {
8 const promise = AccountModel.load(parseInt(id + '', 10))
9
10 return doesAccountExist(promise, res, sendNotFound)
11}
12
13function doesLocalAccountNameExist (name: string, res: Response, sendNotFound = true) {
14 const promise = AccountModel.loadLocalByName(name)
15
16 return doesAccountExist(promise, res, sendNotFound)
17}
18
19function doesAccountNameWithHostExist (nameWithDomain: string, res: Response, sendNotFound = true) {
20 const promise = AccountModel.loadByNameWithHost(nameWithDomain)
21
22 return doesAccountExist(promise, res, sendNotFound)
23}
24
25async function doesAccountExist (p: Promise<MAccountDefault>, res: Response, sendNotFound: boolean) {
26 const account = await p
27
28 if (!account) {
29 if (sendNotFound === true) {
30 res.status(HttpStatusCode.NOT_FOUND_404)
31 .json({ error: 'Account not found' })
32 }
33
34 return false
35 }
36
37 res.locals.account = account
38
39 return true
40}
41
42async function doesUserFeedTokenCorrespond (id: number, token: string, res: Response) {
43 const user = await UserModel.loadByIdWithChannels(parseInt(id + '', 10))
44
45 if (token !== user.feedToken) {
46 res.status(HttpStatusCode.FORBIDDEN_403)
47 .json({ error: 'User and token mismatch' })
48
49 return false
50 }
51
52 res.locals.user = user
53
54 return true
55}
56
57// ---------------------------------------------------------------------------
58
59export {
60 doesAccountIdExist,
61 doesLocalAccountNameExist,
62 doesAccountNameWithHostExist,
63 doesAccountExist,
64 doesUserFeedTokenCorrespond
65}
diff --git a/server/helpers/middlewares/index.ts b/server/helpers/middlewares/index.ts
deleted file mode 100644
index f57f3ad31..000000000
--- a/server/helpers/middlewares/index.ts
+++ /dev/null
@@ -1,7 +0,0 @@
1export * from './abuses'
2export * from './accounts'
3export * from './video-blacklists'
4export * from './video-captions'
5export * from './video-channels'
6export * from './video-playlists'
7export * from './videos'
diff --git a/server/helpers/middlewares/video-blacklists.ts b/server/helpers/middlewares/video-blacklists.ts
deleted file mode 100644
index eda1324d3..000000000
--- a/server/helpers/middlewares/video-blacklists.ts
+++ /dev/null
@@ -1,24 +0,0 @@
1import { Response } from 'express'
2import { VideoBlacklistModel } from '../../models/video/video-blacklist'
3import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
4
5async function doesVideoBlacklistExist (videoId: number, res: Response) {
6 const videoBlacklist = await VideoBlacklistModel.loadByVideoId(videoId)
7
8 if (videoBlacklist === null) {
9 res.status(HttpStatusCode.NOT_FOUND_404)
10 .json({ error: 'Blacklisted video not found' })
11 .end()
12
13 return false
14 }
15
16 res.locals.videoBlacklist = videoBlacklist
17 return true
18}
19
20// ---------------------------------------------------------------------------
21
22export {
23 doesVideoBlacklistExist
24}
diff --git a/server/helpers/middlewares/video-captions.ts b/server/helpers/middlewares/video-captions.ts
deleted file mode 100644
index 226d3c5f8..000000000
--- a/server/helpers/middlewares/video-captions.ts
+++ /dev/null
@@ -1,24 +0,0 @@
1import { Response } from 'express'
2import { VideoCaptionModel } from '../../models/video/video-caption'
3import { MVideoId } from '@server/types/models'
4import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
5
6async function doesVideoCaptionExist (video: MVideoId, language: string, res: Response) {
7 const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(video.id, language)
8
9 if (!videoCaption) {
10 res.status(HttpStatusCode.NOT_FOUND_404)
11 .json({ error: 'Video caption not found' })
12
13 return false
14 }
15
16 res.locals.videoCaption = videoCaption
17 return true
18}
19
20// ---------------------------------------------------------------------------
21
22export {
23 doesVideoCaptionExist
24}
diff --git a/server/helpers/middlewares/video-channels.ts b/server/helpers/middlewares/video-channels.ts
deleted file mode 100644
index e6eab65a2..000000000
--- a/server/helpers/middlewares/video-channels.ts
+++ /dev/null
@@ -1,42 +0,0 @@
1import * as express from 'express'
2import { MChannelBannerAccountDefault } from '@server/types/models'
3import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
4import { VideoChannelModel } from '../../models/video/video-channel'
5
6async function doesLocalVideoChannelNameExist (name: string, res: express.Response) {
7 const videoChannel = await VideoChannelModel.loadLocalByNameAndPopulateAccount(name)
8
9 return processVideoChannelExist(videoChannel, res)
10}
11
12async function doesVideoChannelIdExist (id: number, res: express.Response) {
13 const videoChannel = await VideoChannelModel.loadAndPopulateAccount(+id)
14
15 return processVideoChannelExist(videoChannel, res)
16}
17
18async function doesVideoChannelNameWithHostExist (nameWithDomain: string, res: express.Response) {
19 const videoChannel = await VideoChannelModel.loadByNameWithHostAndPopulateAccount(nameWithDomain)
20
21 return processVideoChannelExist(videoChannel, res)
22}
23
24// ---------------------------------------------------------------------------
25
26export {
27 doesLocalVideoChannelNameExist,
28 doesVideoChannelIdExist,
29 doesVideoChannelNameWithHostExist
30}
31
32function processVideoChannelExist (videoChannel: MChannelBannerAccountDefault, res: express.Response) {
33 if (!videoChannel) {
34 res.status(HttpStatusCode.NOT_FOUND_404)
35 .json({ error: 'Video channel not found' })
36
37 return false
38 }
39
40 res.locals.videoChannel = videoChannel
41 return true
42}
diff --git a/server/helpers/middlewares/video-playlists.ts b/server/helpers/middlewares/video-playlists.ts
deleted file mode 100644
index d2dd80a35..000000000
--- a/server/helpers/middlewares/video-playlists.ts
+++ /dev/null
@@ -1,39 +0,0 @@
1import * as express from 'express'
2import { VideoPlaylistModel } from '../../models/video/video-playlist'
3import { MVideoPlaylist } from '../../types/models/video/video-playlist'
4import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
5
6export type VideoPlaylistFetchType = 'summary' | 'all'
7async function doesVideoPlaylistExist (id: number | string, res: express.Response, fetchType: VideoPlaylistFetchType = 'summary') {
8 if (fetchType === 'summary') {
9 const videoPlaylist = await VideoPlaylistModel.loadWithAccountAndChannelSummary(id, undefined)
10 res.locals.videoPlaylistSummary = videoPlaylist
11
12 return handleVideoPlaylist(videoPlaylist, res)
13 }
14
15 const videoPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(id, undefined)
16 res.locals.videoPlaylistFull = videoPlaylist
17
18 return handleVideoPlaylist(videoPlaylist, res)
19}
20
21// ---------------------------------------------------------------------------
22
23export {
24 doesVideoPlaylistExist
25}
26
27// ---------------------------------------------------------------------------
28
29function handleVideoPlaylist (videoPlaylist: MVideoPlaylist, res: express.Response) {
30 if (!videoPlaylist) {
31 res.status(HttpStatusCode.NOT_FOUND_404)
32 .json({ error: 'Video playlist not found' })
33 .end()
34
35 return false
36 }
37
38 return true
39}
diff --git a/server/helpers/middlewares/videos.ts b/server/helpers/middlewares/videos.ts
deleted file mode 100644
index 403cae092..000000000
--- a/server/helpers/middlewares/videos.ts
+++ /dev/null
@@ -1,125 +0,0 @@
1import { Response } from 'express'
2import { fetchVideo, VideoFetchType } from '../video'
3import { UserRight } from '../../../shared/models/users'
4import { VideoChannelModel } from '../../models/video/video-channel'
5import {
6 MUser,
7 MUserAccountId,
8 MVideoAccountLight,
9 MVideoFullLight,
10 MVideoIdThumbnail,
11 MVideoImmutable,
12 MVideoThumbnail,
13 MVideoWithRights
14} from '@server/types/models'
15import { VideoFileModel } from '@server/models/video/video-file'
16import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
17
18async function doesVideoExist (id: number | string, res: Response, fetchType: VideoFetchType = 'all') {
19 const userId = res.locals.oauth ? res.locals.oauth.token.User.id : undefined
20
21 const video = await fetchVideo(id, fetchType, userId)
22
23 if (video === null) {
24 res.status(HttpStatusCode.NOT_FOUND_404)
25 .json({ error: 'Video not found' })
26 .end()
27
28 return false
29 }
30
31 switch (fetchType) {
32 case 'all':
33 res.locals.videoAll = video as MVideoFullLight
34 break
35
36 case 'only-immutable-attributes':
37 res.locals.onlyImmutableVideo = video as MVideoImmutable
38 break
39
40 case 'id':
41 res.locals.videoId = video as MVideoIdThumbnail
42 break
43
44 case 'only-video':
45 res.locals.onlyVideo = video as MVideoThumbnail
46 break
47
48 case 'only-video-with-rights':
49 res.locals.onlyVideoWithRights = video as MVideoWithRights
50 break
51 }
52
53 return true
54}
55
56async function doesVideoFileOfVideoExist (id: number, videoIdOrUUID: number | string, res: Response) {
57 if (!await VideoFileModel.doesVideoExistForVideoFile(id, videoIdOrUUID)) {
58 res.status(HttpStatusCode.NOT_FOUND_404)
59 .json({ error: 'VideoFile matching Video not found' })
60 .end()
61
62 return false
63 }
64
65 return true
66}
67
68async function doesVideoChannelOfAccountExist (channelId: number, user: MUserAccountId, res: Response) {
69 const videoChannel = await VideoChannelModel.loadAndPopulateAccount(channelId)
70
71 if (videoChannel === null) {
72 res.status(HttpStatusCode.BAD_REQUEST_400)
73 .json({ error: 'Unknown video "video channel" for this instance.' })
74
75 return false
76 }
77
78 // Don't check account id if the user can update any video
79 if (user.hasRight(UserRight.UPDATE_ANY_VIDEO) === true) {
80 res.locals.videoChannel = videoChannel
81 return true
82 }
83
84 if (videoChannel.Account.id !== user.Account.id) {
85 res.status(HttpStatusCode.BAD_REQUEST_400)
86 .json({ error: 'Unknown video "video channel" for this account.' })
87
88 return false
89 }
90
91 res.locals.videoChannel = videoChannel
92 return true
93}
94
95function checkUserCanManageVideo (user: MUser, video: MVideoAccountLight, right: UserRight, res: Response, onlyOwned = true) {
96 // Retrieve the user who did the request
97 if (onlyOwned && video.isOwned() === false) {
98 res.status(HttpStatusCode.FORBIDDEN_403)
99 .json({ error: 'Cannot manage a video of another server.' })
100 .end()
101 return false
102 }
103
104 // Check if the user can delete the video
105 // The user can delete it if he has the right
106 // Or if s/he is the video's account
107 const account = video.VideoChannel.Account
108 if (user.hasRight(right) === false && account.userId !== user.id) {
109 res.status(HttpStatusCode.FORBIDDEN_403)
110 .json({ error: 'Cannot manage a video of another user.' })
111 .end()
112 return false
113 }
114
115 return true
116}
117
118// ---------------------------------------------------------------------------
119
120export {
121 doesVideoChannelOfAccountExist,
122 doesVideoExist,
123 doesVideoFileOfVideoExist,
124 checkUserCanManageVideo
125}
diff --git a/server/helpers/promise-cache.ts b/server/helpers/promise-cache.ts
new file mode 100644
index 000000000..07e8a9962
--- /dev/null
+++ b/server/helpers/promise-cache.ts
@@ -0,0 +1,21 @@
1export class PromiseCache <A, R> {
2 private readonly running = new Map<string, Promise<R>>()
3
4 constructor (
5 private readonly fn: (arg: A) => Promise<R>,
6 private readonly keyBuilder: (arg: A) => string
7 ) {
8 }
9
10 run (arg: A) {
11 const key = this.keyBuilder(arg)
12
13 if (this.running.has(key)) return this.running.get(key)
14
15 const p = this.fn(arg)
16
17 this.running.set(key, p)
18
19 return p.finally(() => this.running.delete(key))
20 }
21}
diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts
index fd2a56f30..36e69458e 100644
--- a/server/helpers/requests.ts
+++ b/server/helpers/requests.ts
@@ -2,7 +2,7 @@ import { createWriteStream, remove } from 'fs-extra'
2import got, { CancelableRequest, Options as GotOptions, RequestError } from 'got' 2import got, { CancelableRequest, Options as GotOptions, RequestError } from 'got'
3import { join } from 'path' 3import { join } from 'path'
4import { CONFIG } from '../initializers/config' 4import { CONFIG } from '../initializers/config'
5import { ACTIVITY_PUB, PEERTUBE_VERSION, WEBSERVER } from '../initializers/constants' 5import { ACTIVITY_PUB, PEERTUBE_VERSION, REQUEST_TIMEOUT, WEBSERVER } from '../initializers/constants'
6import { pipelinePromise } from './core-utils' 6import { pipelinePromise } from './core-utils'
7import { processImage } from './image-utils' 7import { processImage } from './image-utils'
8import { logger } from './logger' 8import { logger } from './logger'
@@ -24,6 +24,7 @@ type PeerTubeRequestOptions = {
24 key: string 24 key: string
25 headers: string[] 25 headers: string[]
26 } 26 }
27 timeout?: number
27 jsonResponse?: boolean 28 jsonResponse?: boolean
28} & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'> 29} & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
29 30
@@ -92,6 +93,10 @@ const peertubeGot = got.extend({
92 path 93 path
93 }, httpSignatureOptions) 94 }, httpSignatureOptions)
94 } 95 }
96 },
97
98 (options: GotOptions) => {
99 options.timeout = REQUEST_TIMEOUT
95 } 100 }
96 ] 101 ]
97 } 102 }
@@ -180,8 +185,10 @@ function buildGotOptions (options: PeerTubeRequestOptions) {
180 185
181 return { 186 return {
182 method: options.method, 187 method: options.method,
188 dnsCache: true,
183 json: options.json, 189 json: options.json,
184 searchParams: options.searchParams, 190 searchParams: options.searchParams,
191 timeout: options.timeout ?? REQUEST_TIMEOUT,
185 headers, 192 headers,
186 context 193 context
187 } 194 }
diff --git a/server/helpers/signup.ts b/server/helpers/signup.ts
deleted file mode 100644
index ed872539b..000000000
--- a/server/helpers/signup.ts
+++ /dev/null
@@ -1,62 +0,0 @@
1import { UserModel } from '../models/account/user'
2import * as ipaddr from 'ipaddr.js'
3import { CONFIG } from '../initializers/config'
4
5const isCidr = require('is-cidr')
6
7async function isSignupAllowed (): Promise<{ allowed: boolean, errorMessage?: string }> {
8 if (CONFIG.SIGNUP.ENABLED === false) {
9 return { allowed: false }
10 }
11
12 // No limit and signup is enabled
13 if (CONFIG.SIGNUP.LIMIT === -1) {
14 return { allowed: true }
15 }
16
17 const totalUsers = await UserModel.countTotal()
18
19 return { allowed: totalUsers < CONFIG.SIGNUP.LIMIT }
20}
21
22function isSignupAllowedForCurrentIP (ip: string) {
23 if (!ip) return false
24
25 const addr = ipaddr.parse(ip)
26 const excludeList = [ 'blacklist' ]
27 let matched = ''
28
29 // if there is a valid, non-empty whitelist, we exclude all unknown adresses too
30 if (CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr(cidr)).length > 0) {
31 excludeList.push('unknown')
32 }
33
34 if (addr.kind() === 'ipv4') {
35 const addrV4 = ipaddr.IPv4.parse(ip)
36 const rangeList = {
37 whitelist: CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr.v4(cidr))
38 .map(cidr => ipaddr.IPv4.parseCIDR(cidr)),
39 blacklist: CONFIG.SIGNUP.FILTERS.CIDR.BLACKLIST.filter(cidr => isCidr.v4(cidr))
40 .map(cidr => ipaddr.IPv4.parseCIDR(cidr))
41 }
42 matched = ipaddr.subnetMatch(addrV4, rangeList, 'unknown')
43 } else if (addr.kind() === 'ipv6') {
44 const addrV6 = ipaddr.IPv6.parse(ip)
45 const rangeList = {
46 whitelist: CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr.v6(cidr))
47 .map(cidr => ipaddr.IPv6.parseCIDR(cidr)),
48 blacklist: CONFIG.SIGNUP.FILTERS.CIDR.BLACKLIST.filter(cidr => isCidr.v6(cidr))
49 .map(cidr => ipaddr.IPv6.parseCIDR(cidr))
50 }
51 matched = ipaddr.subnetMatch(addrV6, rangeList, 'unknown')
52 }
53
54 return !excludeList.includes(matched)
55}
56
57// ---------------------------------------------------------------------------
58
59export {
60 isSignupAllowed,
61 isSignupAllowedForCurrentIP
62}
diff --git a/server/helpers/uuid.ts b/server/helpers/uuid.ts
new file mode 100644
index 000000000..3eb06c773
--- /dev/null
+++ b/server/helpers/uuid.ts
@@ -0,0 +1,32 @@
1import * as short from 'short-uuid'
2
3const translator = short()
4
5function buildUUID () {
6 return short.uuid()
7}
8
9function uuidToShort (uuid: string) {
10 if (!uuid) return uuid
11
12 return translator.fromUUID(uuid)
13}
14
15function shortToUUID (shortUUID: string) {
16 if (!shortUUID) return shortUUID
17
18 return translator.toUUID(shortUUID)
19}
20
21function isShortUUID (value: string) {
22 if (!value) return false
23
24 return value.length === translator.maxLength
25}
26
27export {
28 buildUUID,
29 uuidToShort,
30 shortToUUID,
31 isShortUUID
32}
diff --git a/server/helpers/video.ts b/server/helpers/video.ts
index 7c510f474..f5f645d3e 100644
--- a/server/helpers/video.ts
+++ b/server/helpers/video.ts
@@ -1,69 +1,10 @@
1import { Response } from 'express' 1import { Response } from 'express'
2import { CONFIG } from '@server/initializers/config' 2import { CONFIG } from '@server/initializers/config'
3import { 3import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models'
4 isStreamingPlaylist,
5 MStreamingPlaylistVideo,
6 MVideo,
7 MVideoAccountLightBlacklistAllFiles,
8 MVideoFullLight,
9 MVideoIdThumbnail,
10 MVideoImmutable,
11 MVideoThumbnail,
12 MVideoWithRights
13} from '@server/types/models'
14import { VideoPrivacy, VideoState } from '@shared/models' 4import { VideoPrivacy, VideoState } from '@shared/models'
15import { VideoModel } from '../models/video/video'
16
17type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none' | 'only-immutable-attributes'
18
19function fetchVideo (id: number | string, fetchType: 'all', userId?: number): Promise<MVideoFullLight>
20function fetchVideo (id: number | string, fetchType: 'only-immutable-attributes'): Promise<MVideoImmutable>
21function fetchVideo (id: number | string, fetchType: 'only-video', userId?: number): Promise<MVideoThumbnail>
22function fetchVideo (id: number | string, fetchType: 'only-video-with-rights', userId?: number): Promise<MVideoWithRights>
23function fetchVideo (id: number | string, fetchType: 'id' | 'none', userId?: number): Promise<MVideoIdThumbnail>
24function fetchVideo (
25 id: number | string,
26 fetchType: VideoFetchType,
27 userId?: number
28): Promise<MVideoFullLight | MVideoThumbnail | MVideoWithRights | MVideoIdThumbnail | MVideoImmutable>
29function fetchVideo (
30 id: number | string,
31 fetchType: VideoFetchType,
32 userId?: number
33): Promise<MVideoFullLight | MVideoThumbnail | MVideoWithRights | MVideoIdThumbnail | MVideoImmutable> {
34 if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId)
35
36 if (fetchType === 'only-immutable-attributes') return VideoModel.loadImmutableAttributes(id)
37
38 if (fetchType === 'only-video-with-rights') return VideoModel.loadWithRights(id)
39
40 if (fetchType === 'only-video') return VideoModel.load(id)
41
42 if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id)
43}
44
45type VideoFetchByUrlType = 'all' | 'only-video' | 'only-immutable-attributes'
46
47function fetchVideoByUrl (url: string, fetchType: 'all'): Promise<MVideoAccountLightBlacklistAllFiles>
48function fetchVideoByUrl (url: string, fetchType: 'only-immutable-attributes'): Promise<MVideoImmutable>
49function fetchVideoByUrl (url: string, fetchType: 'only-video'): Promise<MVideoThumbnail>
50function fetchVideoByUrl (
51 url: string,
52 fetchType: VideoFetchByUrlType
53): Promise<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable>
54function fetchVideoByUrl (
55 url: string,
56 fetchType: VideoFetchByUrlType
57): Promise<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
58 if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url)
59
60 if (fetchType === 'only-immutable-attributes') return VideoModel.loadByUrlImmutableAttributes(url)
61
62 if (fetchType === 'only-video') return VideoModel.loadByUrl(url)
63}
64 5
65function getVideoWithAttributes (res: Response) { 6function getVideoWithAttributes (res: Response) {
66 return res.locals.videoAll || res.locals.onlyVideo || res.locals.onlyVideoWithRights 7 return res.locals.videoAPI || res.locals.videoAll || res.locals.onlyVideo
67} 8}
68 9
69function extractVideo (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) { 10function extractVideo (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
@@ -100,11 +41,7 @@ function getExtFromMimetype (mimeTypes: { [id: string]: string | string[] }, mim
100} 41}
101 42
102export { 43export {
103 VideoFetchType,
104 VideoFetchByUrlType,
105 fetchVideo,
106 getVideoWithAttributes, 44 getVideoWithAttributes,
107 fetchVideoByUrl,
108 extractVideo, 45 extractVideo,
109 getExtFromMimetype, 46 getExtFromMimetype,
110 isStateForFederation, 47 isStateForFederation,
diff --git a/server/helpers/webfinger.ts b/server/helpers/webfinger.ts
deleted file mode 100644
index da7e88077..000000000
--- a/server/helpers/webfinger.ts
+++ /dev/null
@@ -1,67 +0,0 @@
1import * as WebFinger from 'webfinger.js'
2import { WebFingerData } from '../../shared'
3import { ActorModel } from '../models/activitypub/actor'
4import { isTestInstance } from './core-utils'
5import { isActivityPubUrlValid } from './custom-validators/activitypub/misc'
6import { WEBSERVER } from '../initializers/constants'
7import { MActorFull } from '../types/models'
8
9const webfinger = new WebFinger({
10 webfist_fallback: false,
11 tls_only: isTestInstance(),
12 uri_fallback: false,
13 request_timeout: 3000
14})
15
16async function loadActorUrlOrGetFromWebfinger (uriArg: string) {
17 // Handle strings like @toto@example.com
18 const uri = uriArg.startsWith('@') ? uriArg.slice(1) : uriArg
19
20 const [ name, host ] = uri.split('@')
21 let actor: MActorFull
22
23 if (!host || host === WEBSERVER.HOST) {
24 actor = await ActorModel.loadLocalByName(name)
25 } else {
26 actor = await ActorModel.loadByNameAndHost(name, host)
27 }
28
29 if (actor) return actor.url
30
31 return getUrlFromWebfinger(uri)
32}
33
34async function getUrlFromWebfinger (uri: string) {
35 const webfingerData: WebFingerData = await webfingerLookup(uri)
36 return getLinkOrThrow(webfingerData)
37}
38
39// ---------------------------------------------------------------------------
40
41export {
42 getUrlFromWebfinger,
43 loadActorUrlOrGetFromWebfinger
44}
45
46// ---------------------------------------------------------------------------
47
48function getLinkOrThrow (webfingerData: WebFingerData) {
49 if (Array.isArray(webfingerData.links) === false) throw new Error('WebFinger links is not an array.')
50
51 const selfLink = webfingerData.links.find(l => l.rel === 'self')
52 if (selfLink === undefined || isActivityPubUrlValid(selfLink.href) === false) {
53 throw new Error('Cannot find self link or href is not a valid URL.')
54 }
55
56 return selfLink.href
57}
58
59function webfingerLookup (nameWithHost: string) {
60 return new Promise<WebFingerData>((res, rej) => {
61 webfinger.lookup(nameWithHost, (err, p) => {
62 if (err) return rej(err)
63
64 return res(p.object)
65 })
66 })
67}
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
index fac3da6ba..fdd361390 100644
--- a/server/helpers/youtube-dl.ts
+++ b/server/helpers/youtube-dl.ts
@@ -6,7 +6,6 @@ import { CONFIG } from '@server/initializers/config'
6import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' 6import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes'
7import { VideoResolution } from '../../shared/models/videos' 7import { VideoResolution } from '../../shared/models/videos'
8import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants' 8import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants'
9import { getEnabledResolutions } from '../lib/video-transcoding'
10import { peertubeTruncate, pipelinePromise, root } from './core-utils' 9import { peertubeTruncate, pipelinePromise, root } from './core-utils'
11import { isVideoFileExtnameValid } from './custom-validators/videos' 10import { isVideoFileExtnameValid } from './custom-validators/videos'
12import { logger } from './logger' 11import { logger } from './logger'
@@ -35,361 +34,359 @@ const processOptions = {
35 maxBuffer: 1024 * 1024 * 10 // 10MB 34 maxBuffer: 1024 * 1024 * 10 // 10MB
36} 35}
37 36
38function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo> { 37class YoutubeDL {
39 return new Promise<YoutubeDLInfo>((res, rej) => {
40 let args = opts || [ '-j', '--flat-playlist' ]
41 38
42 if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) { 39 constructor (private readonly url: string = '', private readonly enabledResolutions: number[] = []) {
43 args.push('--force-ipv4')
44 }
45 40
46 args = wrapWithProxyOptions(args) 41 }
47 args = [ '-f', getYoutubeDLVideoFormat() ].concat(args)
48 42
49 safeGetYoutubeDL() 43 getYoutubeDLInfo (opts?: string[]): Promise<YoutubeDLInfo> {
50 .then(youtubeDL => { 44 return new Promise<YoutubeDLInfo>((res, rej) => {
51 youtubeDL.getInfo(url, args, processOptions, (err, info) => { 45 let args = opts || []
52 if (err) return rej(err)
53 if (info.is_live === true) return rej(new Error('Cannot download a live streaming.'))
54 46
55 const obj = buildVideoInfo(normalizeObject(info)) 47 if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) {
56 if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video' 48 args.push('--force-ipv4')
49 }
57 50
58 return res(obj) 51 args = this.wrapWithProxyOptions(args)
59 }) 52 args = [ '-f', this.getYoutubeDLVideoFormat() ].concat(args)
60 })
61 .catch(err => rej(err))
62 })
63}
64 53
65function getYoutubeDLSubs (url: string, opts?: object): Promise<YoutubeDLSubs> { 54 YoutubeDL.safeGetYoutubeDL()
66 return new Promise<YoutubeDLSubs>((res, rej) => { 55 .then(youtubeDL => {
67 const cwd = CONFIG.STORAGE.TMP_DIR 56 youtubeDL.getInfo(this.url, args, processOptions, (err, info) => {
68 const options = opts || { all: true, format: 'vtt', cwd } 57 if (err) return rej(err)
69 58 if (info.is_live === true) return rej(new Error('Cannot download a live streaming.'))
70 safeGetYoutubeDL()
71 .then(youtubeDL => {
72 youtubeDL.getSubs(url, options, (err, files) => {
73 if (err) return rej(err)
74 if (!files) return []
75
76 logger.debug('Get subtitles from youtube dl.', { url, files })
77
78 const subtitles = files.reduce((acc, filename) => {
79 const matched = filename.match(/\.([a-z]{2})(-[a-z]+)?\.(vtt|ttml)/i)
80 if (!matched || !matched[1]) return acc
81
82 return [
83 ...acc,
84 {
85 language: matched[1],
86 path: join(cwd, filename),
87 filename
88 }
89 ]
90 }, [])
91 59
92 return res(subtitles) 60 const obj = this.buildVideoInfo(this.normalizeObject(info))
61 if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video'
62
63 return res(obj)
64 })
93 }) 65 })
94 }) 66 .catch(err => rej(err))
95 .catch(err => rej(err)) 67 })
96 }) 68 }
97}
98 69
99function getYoutubeDLVideoFormat () { 70 getYoutubeDLSubs (opts?: object): Promise<YoutubeDLSubs> {
100 /** 71 return new Promise<YoutubeDLSubs>((res, rej) => {
101 * list of format selectors in order or preference 72 const cwd = CONFIG.STORAGE.TMP_DIR
102 * see https://github.com/ytdl-org/youtube-dl#format-selection 73 const options = opts || { all: true, format: 'vtt', cwd }
103 * 74
104 * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope 75 YoutubeDL.safeGetYoutubeDL()
105 * of being able to do a "quick-transcode" 76 .then(youtubeDL => {
106 * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9) 77 youtubeDL.getSubs(this.url, options, (err, files) => {
107 * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback 78 if (err) return rej(err)
108 * 79 if (!files) return []
109 * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499 80
110 **/ 81 logger.debug('Get subtitles from youtube dl.', { url: this.url, files })
111 const enabledResolutions = getEnabledResolutions('vod') 82
112 const resolution = enabledResolutions.length === 0 83 const subtitles = files.reduce((acc, filename) => {
113 ? VideoResolution.H_720P 84 const matched = filename.match(/\.([a-z]{2})(-[a-z]+)?\.(vtt|ttml)/i)
114 : Math.max(...enabledResolutions) 85 if (!matched || !matched[1]) return acc
115 86
116 return [ 87 return [
117 `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1 88 ...acc,
118 `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2 89 {
119 `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]`, // case #3 90 language: matched[1],
120 `bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio`, 91 path: join(cwd, filename),
121 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats 92 filename
122 'best' // Ultimate fallback 93 }
123 ].join('/') 94 ]
124} 95 }, [])
96
97 return res(subtitles)
98 })
99 })
100 .catch(err => rej(err))
101 })
102 }
125 103
126function downloadYoutubeDLVideo (url: string, fileExt: string, timeout: number) { 104 getYoutubeDLVideoFormat () {
127 // Leave empty the extension, youtube-dl will add it 105 /**
128 const pathWithoutExtension = generateVideoImportTmpPath(url, '') 106 * list of format selectors in order or preference
107 * see https://github.com/ytdl-org/youtube-dl#format-selection
108 *
109 * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope
110 * of being able to do a "quick-transcode"
111 * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9)
112 * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback
113 *
114 * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499
115 **/
116 const resolution = this.enabledResolutions.length === 0
117 ? VideoResolution.H_720P
118 : Math.max(...this.enabledResolutions)
119
120 return [
121 `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1
122 `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2
123 `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]`, // case #3
124 `bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio`,
125 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
126 'best' // Ultimate fallback
127 ].join('/')
128 }
129 129
130 let timer 130 downloadYoutubeDLVideo (fileExt: string, timeout: number) {
131 // Leave empty the extension, youtube-dl will add it
132 const pathWithoutExtension = generateVideoImportTmpPath(this.url, '')
131 133
132 logger.info('Importing youtubeDL video %s to %s', url, pathWithoutExtension) 134 let timer
133 135
134 let options = [ '-f', getYoutubeDLVideoFormat(), '-o', pathWithoutExtension ] 136 logger.info('Importing youtubeDL video %s to %s', this.url, pathWithoutExtension)
135 options = wrapWithProxyOptions(options)
136 137
137 if (process.env.FFMPEG_PATH) { 138 let options = [ '-f', this.getYoutubeDLVideoFormat(), '-o', pathWithoutExtension ]
138 options = options.concat([ '--ffmpeg-location', process.env.FFMPEG_PATH ]) 139 options = this.wrapWithProxyOptions(options)
139 }
140 140
141 logger.debug('YoutubeDL options for %s.', url, { options }) 141 if (process.env.FFMPEG_PATH) {
142 options = options.concat([ '--ffmpeg-location', process.env.FFMPEG_PATH ])
143 }
142 144
143 return new Promise<string>((res, rej) => { 145 logger.debug('YoutubeDL options for %s.', this.url, { options })
144 safeGetYoutubeDL()
145 .then(youtubeDL => {
146 youtubeDL.exec(url, options, processOptions, async err => {
147 clearTimeout(timer)
148 146
149 try { 147 return new Promise<string>((res, rej) => {
150 // If youtube-dl did not guess an extension for our file, just use .mp4 as default 148 YoutubeDL.safeGetYoutubeDL()
151 if (await pathExists(pathWithoutExtension)) { 149 .then(youtubeDL => {
152 await move(pathWithoutExtension, pathWithoutExtension + '.mp4') 150 youtubeDL.exec(this.url, options, processOptions, async err => {
153 } 151 clearTimeout(timer)
152
153 try {
154 // If youtube-dl did not guess an extension for our file, just use .mp4 as default
155 if (await pathExists(pathWithoutExtension)) {
156 await move(pathWithoutExtension, pathWithoutExtension + '.mp4')
157 }
154 158
155 const path = await guessVideoPathWithExtension(pathWithoutExtension, fileExt) 159 const path = await this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
156 160
157 if (err) { 161 if (err) {
158 remove(path) 162 remove(path)
159 .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err })) 163 .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err }))
160 164
165 return rej(err)
166 }
167
168 return res(path)
169 } catch (err) {
161 return rej(err) 170 return rej(err)
162 } 171 }
163 172 })
164 return res(path) 173
165 } catch (err) { 174 timer = setTimeout(() => {
166 return rej(err) 175 const err = new Error('YoutubeDL download timeout.')
167 } 176
177 this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
178 .then(path => remove(path))
179 .finally(() => rej(err))
180 .catch(err => {
181 logger.error('Cannot remove file in youtubeDL timeout.', { err })
182 return rej(err)
183 })
184 }, timeout)
168 }) 185 })
186 .catch(err => rej(err))
187 })
188 }
169 189
170 timer = setTimeout(() => { 190 buildOriginallyPublishedAt (obj: any) {
171 const err = new Error('YoutubeDL download timeout.') 191 let originallyPublishedAt: Date = null
172 192
173 guessVideoPathWithExtension(pathWithoutExtension, fileExt) 193 const uploadDateMatcher = /^(\d{4})(\d{2})(\d{2})$/.exec(obj.upload_date)
174 .then(path => remove(path)) 194 if (uploadDateMatcher) {
175 .finally(() => rej(err)) 195 originallyPublishedAt = new Date()
176 .catch(err => { 196 originallyPublishedAt.setHours(0, 0, 0, 0)
177 logger.error('Cannot remove file in youtubeDL timeout.', { err })
178 return rej(err)
179 })
180 }, timeout)
181 })
182 .catch(err => rej(err))
183 })
184}
185
186// Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js
187// We rewrote it to avoid sync calls
188async function updateYoutubeDLBinary () {
189 logger.info('Updating youtubeDL binary.')
190 197
191 const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin') 198 const year = parseInt(uploadDateMatcher[1], 10)
192 const bin = join(binDirectory, 'youtube-dl') 199 // Month starts from 0
193 const detailsPath = join(binDirectory, 'details') 200 const month = parseInt(uploadDateMatcher[2], 10) - 1
194 const url = process.env.YOUTUBE_DL_DOWNLOAD_HOST || 'https://yt-dl.org/downloads/latest/youtube-dl' 201 const day = parseInt(uploadDateMatcher[3], 10)
195 202
196 await ensureDir(binDirectory) 203 originallyPublishedAt.setFullYear(year, month, day)
204 }
197 205
198 try { 206 return originallyPublishedAt
199 const result = await got(url, { followRedirect: false }) 207 }
200 208
201 if (result.statusCode !== HttpStatusCode.FOUND_302) { 209 private async guessVideoPathWithExtension (tmpPath: string, sourceExt: string) {
202 logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode) 210 if (!isVideoFileExtnameValid(sourceExt)) {
203 return 211 throw new Error('Invalid video extension ' + sourceExt)
204 } 212 }
205 213
206 const newUrl = result.headers.location 214 const extensions = [ sourceExt, '.mp4', '.mkv', '.webm' ]
207 const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(newUrl)[1]
208 215
209 const downloadFileStream = got.stream(newUrl) 216 for (const extension of extensions) {
210 const writeStream = createWriteStream(bin, { mode: 493 }) 217 const path = tmpPath + extension
211 218
212 await pipelinePromise( 219 if (await pathExists(path)) return path
213 downloadFileStream, 220 }
214 writeStream
215 )
216
217 const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
218 await writeFile(detailsPath, details, { encoding: 'utf8' })
219 221
220 logger.info('youtube-dl updated to version %s.', newVersion) 222 throw new Error('Cannot guess path of ' + tmpPath)
221 } catch (err) {
222 logger.error('Cannot update youtube-dl.', { err })
223 } 223 }
224}
225 224
226async function safeGetYoutubeDL () { 225 private normalizeObject (obj: any) {
227 let youtubeDL 226 const newObj: any = {}
228 227
229 try { 228 for (const key of Object.keys(obj)) {
230 youtubeDL = require('youtube-dl') 229 // Deprecated key
231 } catch (e) { 230 if (key === 'resolution') continue
232 // Download binary
233 await updateYoutubeDLBinary()
234 youtubeDL = require('youtube-dl')
235 }
236 231
237 return youtubeDL 232 const value = obj[key]
238}
239 233
240function buildOriginallyPublishedAt (obj: any) { 234 if (typeof value === 'string') {
241 let originallyPublishedAt: Date = null 235 newObj[key] = value.normalize()
236 } else {
237 newObj[key] = value
238 }
239 }
242 240
243 const uploadDateMatcher = /^(\d{4})(\d{2})(\d{2})$/.exec(obj.upload_date) 241 return newObj
244 if (uploadDateMatcher) { 242 }
245 originallyPublishedAt = new Date()
246 originallyPublishedAt.setHours(0, 0, 0, 0)
247 243
248 const year = parseInt(uploadDateMatcher[1], 10) 244 private buildVideoInfo (obj: any): YoutubeDLInfo {
249 // Month starts from 0 245 return {
250 const month = parseInt(uploadDateMatcher[2], 10) - 1 246 name: this.titleTruncation(obj.title),
251 const day = parseInt(uploadDateMatcher[3], 10) 247 description: this.descriptionTruncation(obj.description),
248 category: this.getCategory(obj.categories),
249 licence: this.getLicence(obj.license),
250 language: this.getLanguage(obj.language),
251 nsfw: this.isNSFW(obj),
252 tags: this.getTags(obj.tags),
253 thumbnailUrl: obj.thumbnail || undefined,
254 originallyPublishedAt: this.buildOriginallyPublishedAt(obj),
255 ext: obj.ext
256 }
257 }
252 258
253 originallyPublishedAt.setFullYear(year, month, day) 259 private titleTruncation (title: string) {
260 return peertubeTruncate(title, {
261 length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
262 separator: /,? +/,
263 omission: ' […]'
264 })
254 } 265 }
255 266
256 return originallyPublishedAt 267 private descriptionTruncation (description: string) {
257} 268 if (!description || description.length < CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.min) return undefined
258 269
259// --------------------------------------------------------------------------- 270 return peertubeTruncate(description, {
271 length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max,
272 separator: /,? +/,
273 omission: ' […]'
274 })
275 }
260 276
261export { 277 private isNSFW (info: any) {
262 updateYoutubeDLBinary, 278 return info.age_limit && info.age_limit >= 16
263 getYoutubeDLVideoFormat, 279 }
264 downloadYoutubeDLVideo,
265 getYoutubeDLSubs,
266 getYoutubeDLInfo,
267 safeGetYoutubeDL,
268 buildOriginallyPublishedAt
269}
270 280
271// --------------------------------------------------------------------------- 281 private getTags (tags: any) {
282 if (Array.isArray(tags) === false) return []
272 283
273async function guessVideoPathWithExtension (tmpPath: string, sourceExt: string) { 284 return tags
274 if (!isVideoFileExtnameValid(sourceExt)) { 285 .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
275 throw new Error('Invalid video extension ' + sourceExt) 286 .map(t => t.normalize())
287 .slice(0, 5)
276 } 288 }
277 289
278 const extensions = [ sourceExt, '.mp4', '.mkv', '.webm' ] 290 private getLicence (licence: string) {
291 if (!licence) return undefined
279 292
280 for (const extension of extensions) { 293 if (licence.includes('Creative Commons Attribution')) return 1
281 const path = tmpPath + extension
282 294
283 if (await pathExists(path)) return path 295 for (const key of Object.keys(VIDEO_LICENCES)) {
284 } 296 const peertubeLicence = VIDEO_LICENCES[key]
297 if (peertubeLicence.toLowerCase() === licence.toLowerCase()) return parseInt(key, 10)
298 }
285 299
286 throw new Error('Cannot guess path of ' + tmpPath) 300 return undefined
287} 301 }
288 302
289function normalizeObject (obj: any) { 303 private getCategory (categories: string[]) {
290 const newObj: any = {} 304 if (!categories) return undefined
291 305
292 for (const key of Object.keys(obj)) { 306 const categoryString = categories[0]
293 // Deprecated key 307 if (!categoryString || typeof categoryString !== 'string') return undefined
294 if (key === 'resolution') continue
295 308
296 const value = obj[key] 309 if (categoryString === 'News & Politics') return 11
297 310
298 if (typeof value === 'string') { 311 for (const key of Object.keys(VIDEO_CATEGORIES)) {
299 newObj[key] = value.normalize() 312 const category = VIDEO_CATEGORIES[key]
300 } else { 313 if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10)
301 newObj[key] = value
302 } 314 }
303 }
304 315
305 return newObj 316 return undefined
306}
307
308function buildVideoInfo (obj: any): YoutubeDLInfo {
309 return {
310 name: titleTruncation(obj.title),
311 description: descriptionTruncation(obj.description),
312 category: getCategory(obj.categories),
313 licence: getLicence(obj.license),
314 language: getLanguage(obj.language),
315 nsfw: isNSFW(obj),
316 tags: getTags(obj.tags),
317 thumbnailUrl: obj.thumbnail || undefined,
318 originallyPublishedAt: buildOriginallyPublishedAt(obj),
319 ext: obj.ext
320 } 317 }
321}
322 318
323function titleTruncation (title: string) { 319 private getLanguage (language: string) {
324 return peertubeTruncate(title, { 320 return VIDEO_LANGUAGES[language] ? language : undefined
325 length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max, 321 }
326 separator: /,? +/,
327 omission: ' […]'
328 })
329}
330 322
331function descriptionTruncation (description: string) { 323 private wrapWithProxyOptions (options: string[]) {
332 if (!description || description.length < CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.min) return undefined 324 if (CONFIG.IMPORT.VIDEOS.HTTP.PROXY.ENABLED) {
325 logger.debug('Using proxy for YoutubeDL')
333 326
334 return peertubeTruncate(description, { 327 return [ '--proxy', CONFIG.IMPORT.VIDEOS.HTTP.PROXY.URL ].concat(options)
335 length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max, 328 }
336 separator: /,? +/,
337 omission: ' […]'
338 })
339}
340 329
341function isNSFW (info: any) { 330 return options
342 return info.age_limit && info.age_limit >= 16 331 }
343}
344 332
345function getTags (tags: any) { 333 // Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js
346 if (Array.isArray(tags) === false) return [] 334 // We rewrote it to avoid sync calls
335 static async updateYoutubeDLBinary () {
336 logger.info('Updating youtubeDL binary.')
347 337
348 return tags 338 const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin')
349 .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min) 339 const bin = join(binDirectory, 'youtube-dl')
350 .map(t => t.normalize()) 340 const detailsPath = join(binDirectory, 'details')
351 .slice(0, 5) 341 const url = process.env.YOUTUBE_DL_DOWNLOAD_HOST || 'https://yt-dl.org/downloads/latest/youtube-dl'
352}
353 342
354function getLicence (licence: string) { 343 await ensureDir(binDirectory)
355 if (!licence) return undefined
356 344
357 if (licence.includes('Creative Commons Attribution')) return 1 345 try {
346 const result = await got(url, { followRedirect: false })
358 347
359 for (const key of Object.keys(VIDEO_LICENCES)) { 348 if (result.statusCode !== HttpStatusCode.FOUND_302) {
360 const peertubeLicence = VIDEO_LICENCES[key] 349 logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
361 if (peertubeLicence.toLowerCase() === licence.toLowerCase()) return parseInt(key, 10) 350 return
362 } 351 }
363 352
364 return undefined 353 const newUrl = result.headers.location
365} 354 const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(newUrl)[1]
366 355
367function getCategory (categories: string[]) { 356 const downloadFileStream = got.stream(newUrl)
368 if (!categories) return undefined 357 const writeStream = createWriteStream(bin, { mode: 493 })
369 358
370 const categoryString = categories[0] 359 await pipelinePromise(
371 if (!categoryString || typeof categoryString !== 'string') return undefined 360 downloadFileStream,
361 writeStream
362 )
372 363
373 if (categoryString === 'News & Politics') return 11 364 const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
365 await writeFile(detailsPath, details, { encoding: 'utf8' })
374 366
375 for (const key of Object.keys(VIDEO_CATEGORIES)) { 367 logger.info('youtube-dl updated to version %s.', newVersion)
376 const category = VIDEO_CATEGORIES[key] 368 } catch (err) {
377 if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10) 369 logger.error('Cannot update youtube-dl.', { err })
370 }
378 } 371 }
379 372
380 return undefined 373 static async safeGetYoutubeDL () {
381} 374 let youtubeDL
382
383function getLanguage (language: string) {
384 return VIDEO_LANGUAGES[language] ? language : undefined
385}
386 375
387function wrapWithProxyOptions (options: string[]) { 376 try {
388 if (CONFIG.IMPORT.VIDEOS.HTTP.PROXY.ENABLED) { 377 youtubeDL = require('youtube-dl')
389 logger.debug('Using proxy for YoutubeDL') 378 } catch (e) {
379 // Download binary
380 await this.updateYoutubeDLBinary()
381 youtubeDL = require('youtube-dl')
382 }
390 383
391 return [ '--proxy', CONFIG.IMPORT.VIDEOS.HTTP.PROXY.URL ].concat(options) 384 return youtubeDL
392 } 385 }
386}
387
388// ---------------------------------------------------------------------------
393 389
394 return options 390export {
391 YoutubeDL
395} 392}
generated by cgit v1.2.3 (git 2.25.1) at 2025-01-08 14:55:04 +0000