aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/helpers
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2023-07-31 14:34:36 +0200
committerChocobozzz <me@florianbigard.com>2023-08-11 15:02:33 +0200
commit3a4992633ee62d5edfbb484d9c6bcb3cf158489d (patch)
treee4510b39bdac9c318fdb4b47018d08f15368b8f0 /server/helpers
parent04d1da5621d25d59bd5fa1543b725c497bf5d9a8 (diff)
downloadPeerTube-3a4992633ee62d5edfbb484d9c6bcb3cf158489d.tar.gz
PeerTube-3a4992633ee62d5edfbb484d9c6bcb3cf158489d.tar.zst
PeerTube-3a4992633ee62d5edfbb484d9c6bcb3cf158489d.zip
Migrate server to ESM
Sorry for the very big commit that may lead to git log issues and merge conflicts, but it's a major step forward: * Server can be faster at startup because imports() are async and we can easily lazy import big modules * Angular doesn't seem to support ES import (with .js extension), so we had to correctly organize peertube into a monorepo: * Use yarn workspace feature * Use typescript reference projects for dependencies * Shared projects have been moved into "packages", each one is now a node module (with a dedicated package.json/tsconfig.json) * server/tools have been moved into apps/ and is now a dedicated app bundled and published on NPM so users don't have to build peertube cli tools manually * server/tests have been moved into packages/ so we don't compile them every time we want to run the server * Use isolatedModule option: * Had to move from const enum to const (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums) * Had to explictely specify "type" imports when used in decorators * Prefer tsx (that uses esbuild under the hood) instead of ts-node to load typescript files (tests with mocha or scripts): * To reduce test complexity as esbuild doesn't support decorator metadata, we only test server files that do not import server models * We still build tests files into js files for a faster CI * Remove unmaintained peertube CLI import script * Removed some barrels to speed up execution (less imports)
Diffstat (limited to 'server/helpers')
-rw-r--r--server/helpers/actors.ts17
-rw-r--r--server/helpers/audit-logger.ts287
-rw-r--r--server/helpers/captions-utils.ts53
-rw-r--r--server/helpers/core-utils.ts315
-rw-r--r--server/helpers/custom-jsonld-signature.ts91
-rw-r--r--server/helpers/custom-validators/abuses.ts68
-rw-r--r--server/helpers/custom-validators/accounts.ts22
-rw-r--r--server/helpers/custom-validators/activitypub/activity.ts151
-rw-r--r--server/helpers/custom-validators/activitypub/actor.ts142
-rw-r--r--server/helpers/custom-validators/activitypub/cache-file.ts26
-rw-r--r--server/helpers/custom-validators/activitypub/misc.ts76
-rw-r--r--server/helpers/custom-validators/activitypub/playlist.ts29
-rw-r--r--server/helpers/custom-validators/activitypub/signature.ts22
-rw-r--r--server/helpers/custom-validators/activitypub/video-comments.ts59
-rw-r--r--server/helpers/custom-validators/activitypub/videos.ts241
-rw-r--r--server/helpers/custom-validators/activitypub/watch-action.ts37
-rw-r--r--server/helpers/custom-validators/actor-images.ts24
-rw-r--r--server/helpers/custom-validators/bulk.ts9
-rw-r--r--server/helpers/custom-validators/feeds.ts23
-rw-r--r--server/helpers/custom-validators/follows.ts30
-rw-r--r--server/helpers/custom-validators/jobs.ts21
-rw-r--r--server/helpers/custom-validators/logs.ts42
-rw-r--r--server/helpers/custom-validators/metrics.ts10
-rw-r--r--server/helpers/custom-validators/misc.ts190
-rw-r--r--server/helpers/custom-validators/plugins.ts178
-rw-r--r--server/helpers/custom-validators/runners/jobs.ts197
-rw-r--r--server/helpers/custom-validators/runners/runners.ts30
-rw-r--r--server/helpers/custom-validators/search.ts37
-rw-r--r--server/helpers/custom-validators/servers.ts42
-rw-r--r--server/helpers/custom-validators/user-notifications.ts23
-rw-r--r--server/helpers/custom-validators/user-registration.ts25
-rw-r--r--server/helpers/custom-validators/users.ts123
-rw-r--r--server/helpers/custom-validators/video-blacklist.ts22
-rw-r--r--server/helpers/custom-validators/video-captions.ts43
-rw-r--r--server/helpers/custom-validators/video-channel-syncs.ts6
-rw-r--r--server/helpers/custom-validators/video-channels.ts32
-rw-r--r--server/helpers/custom-validators/video-comments.ts14
-rw-r--r--server/helpers/custom-validators/video-imports.ts46
-rw-r--r--server/helpers/custom-validators/video-lives.ts11
-rw-r--r--server/helpers/custom-validators/video-ownership.ts20
-rw-r--r--server/helpers/custom-validators/video-playlists.ts35
-rw-r--r--server/helpers/custom-validators/video-rates.ts5
-rw-r--r--server/helpers/custom-validators/video-redundancies.ts12
-rw-r--r--server/helpers/custom-validators/video-stats.ts16
-rw-r--r--server/helpers/custom-validators/video-studio.ts53
-rw-r--r--server/helpers/custom-validators/video-transcoding.ts12
-rw-r--r--server/helpers/custom-validators/video-view.ts12
-rw-r--r--server/helpers/custom-validators/videos.ts218
-rw-r--r--server/helpers/custom-validators/webfinger.ts21
-rw-r--r--server/helpers/database-utils.ts121
-rw-r--r--server/helpers/debounce.ts16
-rw-r--r--server/helpers/decache.ts78
-rw-r--r--server/helpers/dns.ts29
-rw-r--r--server/helpers/express-utils.ts156
-rw-r--r--server/helpers/ffmpeg/codecs.ts64
-rw-r--r--server/helpers/ffmpeg/ffmpeg-image.ts14
-rw-r--r--server/helpers/ffmpeg/ffmpeg-options.ts45
-rw-r--r--server/helpers/ffmpeg/framerate.ts44
-rw-r--r--server/helpers/ffmpeg/index.ts4
-rw-r--r--server/helpers/geo-ip.ts78
-rw-r--r--server/helpers/image-utils.ts179
-rw-r--r--server/helpers/logger.ts208
-rw-r--r--server/helpers/markdown.ts90
-rw-r--r--server/helpers/memoize.ts12
-rw-r--r--server/helpers/otp.ts58
-rw-r--r--server/helpers/peertube-crypto.ts208
-rw-r--r--server/helpers/promise-cache.ts39
-rw-r--r--server/helpers/proxy.ts14
-rw-r--r--server/helpers/query.ts81
-rw-r--r--server/helpers/regexp.ts22
-rw-r--r--server/helpers/requests.ts246
-rw-r--r--server/helpers/stream-replacer.ts58
-rw-r--r--server/helpers/token-generator.ts19
-rw-r--r--server/helpers/upload.ts14
-rw-r--r--server/helpers/utils.ts70
-rw-r--r--server/helpers/version.ts36
-rw-r--r--server/helpers/video.ts51
-rw-r--r--server/helpers/webtorrent.ts258
-rw-r--r--server/helpers/youtube-dl/index.ts3
-rw-r--r--server/helpers/youtube-dl/youtube-dl-cli.ts259
-rw-r--r--server/helpers/youtube-dl/youtube-dl-info-builder.ts205
-rw-r--r--server/helpers/youtube-dl/youtube-dl-wrapper.ts154
82 files changed, 0 insertions, 6151 deletions
diff --git a/server/helpers/actors.ts b/server/helpers/actors.ts
deleted file mode 100644
index c31fe6f8e..000000000
--- a/server/helpers/actors.ts
+++ /dev/null
@@ -1,17 +0,0 @@
1import { WEBSERVER } from '@server/initializers/constants'
2
3function handleToNameAndHost (handle: string) {
4 let [ name, host ] = handle.split('@')
5 if (host === WEBSERVER.HOST) host = null
6
7 return { name, host, handle }
8}
9
10function handlesToNameAndHost (handles: string[]) {
11 return handles.map(h => handleToNameAndHost(h))
12}
13
14export {
15 handleToNameAndHost,
16 handlesToNameAndHost
17}
diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts
deleted file mode 100644
index 7e8a03e8f..000000000
--- a/server/helpers/audit-logger.ts
+++ /dev/null
@@ -1,287 +0,0 @@
1import { diff } from 'deep-object-diff'
2import express from 'express'
3import flatten from 'flat'
4import { chain } from 'lodash'
5import { join } from 'path'
6import { addColors, config, createLogger, format, transports } from 'winston'
7import { AUDIT_LOG_FILENAME } from '@server/initializers/constants'
8import { AdminAbuse, CustomConfig, User, VideoChannel, VideoChannelSync, VideoComment, VideoDetails, VideoImport } from '@shared/models'
9import { CONFIG } from '../initializers/config'
10import { jsonLoggerFormat, labelFormatter } from './logger'
11
12function getAuditIdFromRes (res: express.Response) {
13 return res.locals.oauth.token.User.username
14}
15
16enum AUDIT_TYPE {
17 CREATE = 'create',
18 UPDATE = 'update',
19 DELETE = 'delete'
20}
21
22const colors = config.npm.colors
23colors.audit = config.npm.colors.info
24
25addColors(colors)
26
27const auditLogger = createLogger({
28 levels: { audit: 0 },
29 transports: [
30 new transports.File({
31 filename: join(CONFIG.STORAGE.LOG_DIR, AUDIT_LOG_FILENAME),
32 level: 'audit',
33 maxsize: 5242880,
34 maxFiles: 5,
35 format: format.combine(
36 format.timestamp(),
37 labelFormatter(),
38 format.splat(),
39 jsonLoggerFormat
40 )
41 })
42 ],
43 exitOnError: true
44})
45
46function auditLoggerWrapper (domain: string, user: string, action: AUDIT_TYPE, entity: EntityAuditView, oldEntity: EntityAuditView = null) {
47 let entityInfos: object
48 if (action === AUDIT_TYPE.UPDATE && oldEntity) {
49 const oldEntityKeys = oldEntity.toLogKeys()
50 const diffObject = diff(oldEntityKeys, entity.toLogKeys())
51 const diffKeys = Object.entries(diffObject).reduce((newKeys, entry) => {
52 newKeys[`new-${entry[0]}`] = entry[1]
53 return newKeys
54 }, {})
55 entityInfos = { ...oldEntityKeys, ...diffKeys }
56 } else {
57 entityInfos = { ...entity.toLogKeys() }
58 }
59 auditLogger.log('audit', JSON.stringify({
60 user,
61 domain,
62 action,
63 ...entityInfos
64 }))
65}
66
67function auditLoggerFactory (domain: string) {
68 return {
69 create (user: string, entity: EntityAuditView) {
70 auditLoggerWrapper(domain, user, AUDIT_TYPE.CREATE, entity)
71 },
72 update (user: string, entity: EntityAuditView, oldEntity: EntityAuditView) {
73 auditLoggerWrapper(domain, user, AUDIT_TYPE.UPDATE, entity, oldEntity)
74 },
75 delete (user: string, entity: EntityAuditView) {
76 auditLoggerWrapper(domain, user, AUDIT_TYPE.DELETE, entity)
77 }
78 }
79}
80
81abstract class EntityAuditView {
82 constructor (private readonly keysToKeep: string[], private readonly prefix: string, private readonly entityInfos: object) { }
83
84 toLogKeys (): object {
85 return chain(flatten<object, any>(this.entityInfos, { delimiter: '-', safe: true }))
86 .pick(this.keysToKeep)
87 .mapKeys((_value, key) => `${this.prefix}-${key}`)
88 .value()
89 }
90}
91
92const videoKeysToKeep = [
93 'tags',
94 'uuid',
95 'id',
96 'uuid',
97 'createdAt',
98 'updatedAt',
99 'publishedAt',
100 'category',
101 'licence',
102 'language',
103 'privacy',
104 'description',
105 'duration',
106 'isLocal',
107 'name',
108 'thumbnailPath',
109 'previewPath',
110 'nsfw',
111 'waitTranscoding',
112 'account-id',
113 'account-uuid',
114 'account-name',
115 'channel-id',
116 'channel-uuid',
117 'channel-name',
118 'support',
119 'commentsEnabled',
120 'downloadEnabled'
121]
122class VideoAuditView extends EntityAuditView {
123 constructor (video: VideoDetails) {
124 super(videoKeysToKeep, 'video', video)
125 }
126}
127
128const videoImportKeysToKeep = [
129 'id',
130 'targetUrl',
131 'video-name'
132]
133class VideoImportAuditView extends EntityAuditView {
134 constructor (videoImport: VideoImport) {
135 super(videoImportKeysToKeep, 'video-import', videoImport)
136 }
137}
138
139const commentKeysToKeep = [
140 'id',
141 'text',
142 'threadId',
143 'inReplyToCommentId',
144 'videoId',
145 'createdAt',
146 'updatedAt',
147 'totalReplies',
148 'account-id',
149 'account-uuid',
150 'account-name'
151]
152class CommentAuditView extends EntityAuditView {
153 constructor (comment: VideoComment) {
154 super(commentKeysToKeep, 'comment', comment)
155 }
156}
157
158const userKeysToKeep = [
159 'id',
160 'username',
161 'email',
162 'nsfwPolicy',
163 'autoPlayVideo',
164 'role',
165 'videoQuota',
166 'createdAt',
167 'account-id',
168 'account-uuid',
169 'account-name',
170 'account-followingCount',
171 'account-followersCount',
172 'account-createdAt',
173 'account-updatedAt',
174 'account-avatar-path',
175 'account-avatar-createdAt',
176 'account-avatar-updatedAt',
177 'account-displayName',
178 'account-description',
179 'videoChannels'
180]
181class UserAuditView extends EntityAuditView {
182 constructor (user: User) {
183 super(userKeysToKeep, 'user', user)
184 }
185}
186
187const channelKeysToKeep = [
188 'id',
189 'uuid',
190 'name',
191 'followingCount',
192 'followersCount',
193 'createdAt',
194 'updatedAt',
195 'avatar-path',
196 'avatar-createdAt',
197 'avatar-updatedAt',
198 'displayName',
199 'description',
200 'support',
201 'isLocal',
202 'ownerAccount-id',
203 'ownerAccount-uuid',
204 'ownerAccount-name',
205 'ownerAccount-displayedName'
206]
207class VideoChannelAuditView extends EntityAuditView {
208 constructor (channel: VideoChannel) {
209 super(channelKeysToKeep, 'channel', channel)
210 }
211}
212
213const abuseKeysToKeep = [
214 'id',
215 'reason',
216 'reporterAccount',
217 'createdAt'
218]
219class AbuseAuditView extends EntityAuditView {
220 constructor (abuse: AdminAbuse) {
221 super(abuseKeysToKeep, 'abuse', abuse)
222 }
223}
224
225const customConfigKeysToKeep = [
226 'instance-name',
227 'instance-shortDescription',
228 'instance-description',
229 'instance-terms',
230 'instance-defaultClientRoute',
231 'instance-defaultNSFWPolicy',
232 'instance-customizations-javascript',
233 'instance-customizations-css',
234 'services-twitter-username',
235 'services-twitter-whitelisted',
236 'cache-previews-size',
237 'cache-captions-size',
238 'signup-enabled',
239 'signup-limit',
240 'signup-requiresEmailVerification',
241 'admin-email',
242 'user-videoQuota',
243 'transcoding-enabled',
244 'transcoding-threads',
245 'transcoding-resolutions'
246]
247class CustomConfigAuditView extends EntityAuditView {
248 constructor (customConfig: CustomConfig) {
249 const infos: any = customConfig
250 const resolutionsDict = infos.transcoding.resolutions
251 const resolutionsArray = []
252
253 Object.entries(resolutionsDict)
254 .forEach(([ resolution, isEnabled ]) => {
255 if (isEnabled) resolutionsArray.push(resolution)
256 })
257
258 Object.assign({}, infos, { transcoding: { resolutions: resolutionsArray } })
259 super(customConfigKeysToKeep, 'config', infos)
260 }
261}
262
263const channelSyncKeysToKeep = [
264 'id',
265 'externalChannelUrl',
266 'channel-id',
267 'channel-name'
268]
269class VideoChannelSyncAuditView extends EntityAuditView {
270 constructor (channelSync: VideoChannelSync) {
271 super(channelSyncKeysToKeep, 'channelSync', channelSync)
272 }
273}
274
275export {
276 getAuditIdFromRes,
277
278 auditLoggerFactory,
279 VideoImportAuditView,
280 VideoChannelAuditView,
281 CommentAuditView,
282 UserAuditView,
283 VideoAuditView,
284 AbuseAuditView,
285 CustomConfigAuditView,
286 VideoChannelSyncAuditView
287}
diff --git a/server/helpers/captions-utils.ts b/server/helpers/captions-utils.ts
deleted file mode 100644
index f6e5b9784..000000000
--- a/server/helpers/captions-utils.ts
+++ /dev/null
@@ -1,53 +0,0 @@
1import { createReadStream, createWriteStream, move, remove } from 'fs-extra'
2import { join } from 'path'
3import srt2vtt from 'srt-to-vtt'
4import { Transform } from 'stream'
5import { MVideoCaption } from '@server/types/models'
6import { CONFIG } from '../initializers/config'
7import { pipelinePromise } from './core-utils'
8
9async function moveAndProcessCaptionFile (physicalFile: { filename: string, path: string }, videoCaption: MVideoCaption) {
10 const videoCaptionsDir = CONFIG.STORAGE.CAPTIONS_DIR
11 const destination = join(videoCaptionsDir, videoCaption.filename)
12
13 // Convert this srt file to vtt
14 if (physicalFile.path.endsWith('.srt')) {
15 await convertSrtToVtt(physicalFile.path, destination)
16 await remove(physicalFile.path)
17 } else if (physicalFile.path !== destination) { // Just move the vtt file
18 await move(physicalFile.path, destination, { overwrite: true })
19 }
20
21 // This is important in case if there is another attempt in the retry process
22 physicalFile.filename = videoCaption.filename
23 physicalFile.path = destination
24}
25
26// ---------------------------------------------------------------------------
27
28export {
29 moveAndProcessCaptionFile
30}
31
32// ---------------------------------------------------------------------------
33
34function convertSrtToVtt (source: string, destination: string) {
35 const fixVTT = new Transform({
36 transform: (chunk, _encoding, cb) => {
37 let block: string = chunk.toString()
38
39 block = block.replace(/(\d\d:\d\d:\d\d)(\s)/g, '$1.000$2')
40 .replace(/(\d\d:\d\d:\d\d),(\d)(\s)/g, '$1.00$2$3')
41 .replace(/(\d\d:\d\d:\d\d),(\d\d)(\s)/g, '$1.0$2$3')
42
43 return cb(undefined, block)
44 }
45 })
46
47 return pipelinePromise(
48 createReadStream(source),
49 srt2vtt(),
50 fixVTT,
51 createWriteStream(destination)
52 )
53}
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
deleted file mode 100644
index 242c49e89..000000000
--- a/server/helpers/core-utils.ts
+++ /dev/null
@@ -1,315 +0,0 @@
1/* eslint-disable no-useless-call */
2
3/*
4 Different from 'utils' because we don't import other PeerTube modules.
5 Useful to avoid circular dependencies.
6*/
7
8import { exec, ExecOptions } from 'child_process'
9import { ED25519KeyPairOptions, generateKeyPair, randomBytes, RSAKeyPairOptions, scrypt } from 'crypto'
10import { truncate } from 'lodash'
11import { pipeline } from 'stream'
12import { URL } from 'url'
13import { promisify } from 'util'
14import { promisify1, promisify2, promisify3 } from '@shared/core-utils'
15
16const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => {
17 if (!oldObject || typeof oldObject !== 'object') {
18 return valueConverter(oldObject)
19 }
20
21 if (Array.isArray(oldObject)) {
22 return oldObject.map(e => objectConverter(e, keyConverter, valueConverter))
23 }
24
25 const newObject = {}
26 Object.keys(oldObject).forEach(oldKey => {
27 const newKey = keyConverter(oldKey)
28 newObject[newKey] = objectConverter(oldObject[oldKey], keyConverter, valueConverter)
29 })
30
31 return newObject
32}
33
34function mapToJSON (map: Map<any, any>) {
35 const obj: any = {}
36
37 for (const [ k, v ] of map) {
38 obj[k] = v
39 }
40
41 return obj
42}
43
44// ---------------------------------------------------------------------------
45
46const timeTable = {
47 ms: 1,
48 second: 1000,
49 minute: 60000,
50 hour: 3600000,
51 day: 3600000 * 24,
52 week: 3600000 * 24 * 7,
53 month: 3600000 * 24 * 30
54}
55
56export function parseDurationToMs (duration: number | string): number {
57 if (duration === null) return null
58 if (typeof duration === 'number') return duration
59 if (!isNaN(+duration)) return +duration
60
61 if (typeof duration === 'string') {
62 const split = duration.match(/^([\d.,]+)\s?(\w+)$/)
63
64 if (split.length === 3) {
65 const len = parseFloat(split[1])
66 let unit = split[2].replace(/s$/i, '').toLowerCase()
67 if (unit === 'm') {
68 unit = 'ms'
69 }
70
71 return (len || 1) * (timeTable[unit] || 0)
72 }
73 }
74
75 throw new Error(`Duration ${duration} could not be properly parsed`)
76}
77
78export function parseBytes (value: string | number): number {
79 if (typeof value === 'number') return value
80 if (!isNaN(+value)) return +value
81
82 const tgm = /^(\d+)\s*TB\s*(\d+)\s*GB\s*(\d+)\s*MB$/
83 const tg = /^(\d+)\s*TB\s*(\d+)\s*GB$/
84 const tm = /^(\d+)\s*TB\s*(\d+)\s*MB$/
85 const gm = /^(\d+)\s*GB\s*(\d+)\s*MB$/
86 const t = /^(\d+)\s*TB$/
87 const g = /^(\d+)\s*GB$/
88 const m = /^(\d+)\s*MB$/
89 const b = /^(\d+)\s*B$/
90
91 let match: RegExpMatchArray
92
93 if (value.match(tgm)) {
94 match = value.match(tgm)
95 return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 +
96 parseInt(match[2], 10) * 1024 * 1024 * 1024 +
97 parseInt(match[3], 10) * 1024 * 1024
98 }
99
100 if (value.match(tg)) {
101 match = value.match(tg)
102 return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 +
103 parseInt(match[2], 10) * 1024 * 1024 * 1024
104 }
105
106 if (value.match(tm)) {
107 match = value.match(tm)
108 return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 +
109 parseInt(match[2], 10) * 1024 * 1024
110 }
111
112 if (value.match(gm)) {
113 match = value.match(gm)
114 return parseInt(match[1], 10) * 1024 * 1024 * 1024 +
115 parseInt(match[2], 10) * 1024 * 1024
116 }
117
118 if (value.match(t)) {
119 match = value.match(t)
120 return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024
121 }
122
123 if (value.match(g)) {
124 match = value.match(g)
125 return parseInt(match[1], 10) * 1024 * 1024 * 1024
126 }
127
128 if (value.match(m)) {
129 match = value.match(m)
130 return parseInt(match[1], 10) * 1024 * 1024
131 }
132
133 if (value.match(b)) {
134 match = value.match(b)
135 return parseInt(match[1], 10) * 1024
136 }
137
138 return parseInt(value, 10)
139}
140
141// ---------------------------------------------------------------------------
142
143function sanitizeUrl (url: string) {
144 const urlObject = new URL(url)
145
146 if (urlObject.protocol === 'https:' && urlObject.port === '443') {
147 urlObject.port = ''
148 } else if (urlObject.protocol === 'http:' && urlObject.port === '80') {
149 urlObject.port = ''
150 }
151
152 return urlObject.href.replace(/\/$/, '')
153}
154
155// Don't import remote scheme from constants because we are in core utils
156function sanitizeHost (host: string, remoteScheme: string) {
157 const toRemove = remoteScheme === 'https' ? 443 : 80
158
159 return host.replace(new RegExp(`:${toRemove}$`), '')
160}
161
162// ---------------------------------------------------------------------------
163
164function isTestInstance () {
165 return process.env.NODE_ENV === 'test'
166}
167
168function isDevInstance () {
169 return process.env.NODE_ENV === 'dev'
170}
171
172function isTestOrDevInstance () {
173 return isTestInstance() || isDevInstance()
174}
175
176function isProdInstance () {
177 return process.env.NODE_ENV === 'production'
178}
179
180function getAppNumber () {
181 return process.env.NODE_APP_INSTANCE || ''
182}
183
184// ---------------------------------------------------------------------------
185
186// Consistent with .length, lodash truncate function is not
187function peertubeTruncate (str: string, options: { length: number, separator?: RegExp, omission?: string }) {
188 const truncatedStr = truncate(str, options)
189
190 // The truncated string is okay, we can return it
191 if (truncatedStr.length <= options.length) return truncatedStr
192
193 // Lodash takes into account all UTF characters, whereas String.prototype.length does not: some characters have a length of 2
194 // We always use the .length so we need to truncate more if needed
195 options.length -= truncatedStr.length - options.length
196 return truncate(str, options)
197}
198
199function pageToStartAndCount (page: number, itemsPerPage: number) {
200 const start = (page - 1) * itemsPerPage
201
202 return { start, count: itemsPerPage }
203}
204
205// ---------------------------------------------------------------------------
206
207type SemVersion = { major: number, minor: number, patch: number }
208function parseSemVersion (s: string) {
209 const parsed = s.match(/^v?(\d+)\.(\d+)\.(\d+)$/i)
210
211 return {
212 major: parseInt(parsed[1]),
213 minor: parseInt(parsed[2]),
214 patch: parseInt(parsed[3])
215 } as SemVersion
216}
217
218// ---------------------------------------------------------------------------
219
220function execShell (command: string, options?: ExecOptions) {
221 return new Promise<{ err?: Error, stdout: string, stderr: string }>((res, rej) => {
222 exec(command, options, (err, stdout, stderr) => {
223 // eslint-disable-next-line prefer-promise-reject-errors
224 if (err) return rej({ err, stdout, stderr })
225
226 return res({ stdout, stderr })
227 })
228 })
229}
230
231// ---------------------------------------------------------------------------
232
233function generateRSAKeyPairPromise (size: number) {
234 return new Promise<{ publicKey: string, privateKey: string }>((res, rej) => {
235 const options: RSAKeyPairOptions<'pem', 'pem'> = {
236 modulusLength: size,
237 publicKeyEncoding: {
238 type: 'spki',
239 format: 'pem'
240 },
241 privateKeyEncoding: {
242 type: 'pkcs1',
243 format: 'pem'
244 }
245 }
246
247 generateKeyPair('rsa', options, (err, publicKey, privateKey) => {
248 if (err) return rej(err)
249
250 return res({ publicKey, privateKey })
251 })
252 })
253}
254
255function generateED25519KeyPairPromise () {
256 return new Promise<{ publicKey: string, privateKey: string }>((res, rej) => {
257 const options: ED25519KeyPairOptions<'pem', 'pem'> = {
258 publicKeyEncoding: {
259 type: 'spki',
260 format: 'pem'
261 },
262 privateKeyEncoding: {
263 type: 'pkcs8',
264 format: 'pem'
265 }
266 }
267
268 generateKeyPair('ed25519', options, (err, publicKey, privateKey) => {
269 if (err) return rej(err)
270
271 return res({ publicKey, privateKey })
272 })
273 })
274}
275
276// ---------------------------------------------------------------------------
277
278const randomBytesPromise = promisify1<number, Buffer>(randomBytes)
279const scryptPromise = promisify3<string, string, number, Buffer>(scrypt)
280const execPromise2 = promisify2<string, any, string>(exec)
281const execPromise = promisify1<string, string>(exec)
282const pipelinePromise = promisify(pipeline)
283
284// ---------------------------------------------------------------------------
285
286export {
287 isTestInstance,
288 isTestOrDevInstance,
289 isProdInstance,
290 getAppNumber,
291
292 objectConverter,
293 mapToJSON,
294
295 sanitizeUrl,
296 sanitizeHost,
297
298 execShell,
299
300 pageToStartAndCount,
301 peertubeTruncate,
302
303 scryptPromise,
304
305 randomBytesPromise,
306
307 generateRSAKeyPairPromise,
308 generateED25519KeyPairPromise,
309
310 execPromise2,
311 execPromise,
312 pipelinePromise,
313
314 parseSemVersion
315}
diff --git a/server/helpers/custom-jsonld-signature.ts b/server/helpers/custom-jsonld-signature.ts
deleted file mode 100644
index 3c706e372..000000000
--- a/server/helpers/custom-jsonld-signature.ts
+++ /dev/null
@@ -1,91 +0,0 @@
1import AsyncLRU from 'async-lru'
2import { logger } from './logger'
3
4import jsonld = require('jsonld')
5
6const CACHE = {
7 'https://w3id.org/security/v1': {
8 '@context': {
9 id: '@id',
10 type: '@type',
11
12 dc: 'http://purl.org/dc/terms/',
13 sec: 'https://w3id.org/security#',
14 xsd: 'http://www.w3.org/2001/XMLSchema#',
15
16 EcdsaKoblitzSignature2016: 'sec:EcdsaKoblitzSignature2016',
17 Ed25519Signature2018: 'sec:Ed25519Signature2018',
18 EncryptedMessage: 'sec:EncryptedMessage',
19 GraphSignature2012: 'sec:GraphSignature2012',
20 LinkedDataSignature2015: 'sec:LinkedDataSignature2015',
21 LinkedDataSignature2016: 'sec:LinkedDataSignature2016',
22 CryptographicKey: 'sec:Key',
23
24 authenticationTag: 'sec:authenticationTag',
25 canonicalizationAlgorithm: 'sec:canonicalizationAlgorithm',
26 cipherAlgorithm: 'sec:cipherAlgorithm',
27 cipherData: 'sec:cipherData',
28 cipherKey: 'sec:cipherKey',
29 created: { '@id': 'dc:created', '@type': 'xsd:dateTime' },
30 creator: { '@id': 'dc:creator', '@type': '@id' },
31 digestAlgorithm: 'sec:digestAlgorithm',
32 digestValue: 'sec:digestValue',
33 domain: 'sec:domain',
34 encryptionKey: 'sec:encryptionKey',
35 expiration: { '@id': 'sec:expiration', '@type': 'xsd:dateTime' },
36 expires: { '@id': 'sec:expiration', '@type': 'xsd:dateTime' },
37 initializationVector: 'sec:initializationVector',
38 iterationCount: 'sec:iterationCount',
39 nonce: 'sec:nonce',
40 normalizationAlgorithm: 'sec:normalizationAlgorithm',
41 owner: { '@id': 'sec:owner', '@type': '@id' },
42 password: 'sec:password',
43 privateKey: { '@id': 'sec:privateKey', '@type': '@id' },
44 privateKeyPem: 'sec:privateKeyPem',
45 publicKey: { '@id': 'sec:publicKey', '@type': '@id' },
46 publicKeyBase58: 'sec:publicKeyBase58',
47 publicKeyPem: 'sec:publicKeyPem',
48 publicKeyWif: 'sec:publicKeyWif',
49 publicKeyService: { '@id': 'sec:publicKeyService', '@type': '@id' },
50 revoked: { '@id': 'sec:revoked', '@type': 'xsd:dateTime' },
51 salt: 'sec:salt',
52 signature: 'sec:signature',
53 signatureAlgorithm: 'sec:signingAlgorithm',
54 signatureValue: 'sec:signatureValue'
55 }
56 }
57}
58
59const nodeDocumentLoader = jsonld.documentLoaders.node()
60
61const lru = new AsyncLRU({
62 max: 10,
63 load: (url, cb) => {
64 if (CACHE[url] !== undefined) {
65 logger.debug('Using cache for JSON-LD %s.', url)
66
67 return cb(null, {
68 contextUrl: null,
69 document: CACHE[url],
70 documentUrl: url
71 })
72 }
73
74 nodeDocumentLoader(url)
75 .then(value => cb(null, value))
76 .catch(err => cb(err))
77 }
78})
79
80/* eslint-disable no-import-assign */
81jsonld.documentLoader = (url) => {
82 return new Promise((res, rej) => {
83 lru.get(url, (err, value) => {
84 if (err) return rej(err)
85
86 return res(value)
87 })
88 })
89}
90
91export { jsonld }
diff --git a/server/helpers/custom-validators/abuses.ts b/server/helpers/custom-validators/abuses.ts
deleted file mode 100644
index 94719641a..000000000
--- a/server/helpers/custom-validators/abuses.ts
+++ /dev/null
@@ -1,68 +0,0 @@
1import validator from 'validator'
2import { abusePredefinedReasonsMap } from '@shared/core-utils/abuse'
3import { AbuseCreate, AbuseFilter, AbusePredefinedReasonsString, AbuseVideoIs } from '@shared/models'
4import { ABUSE_STATES, CONSTRAINTS_FIELDS } from '../../initializers/constants'
5import { exists, isArray } from './misc'
6
7const ABUSES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.ABUSES
8const ABUSE_MESSAGES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.ABUSE_MESSAGES
9
10function isAbuseReasonValid (value: string) {
11 return exists(value) && validator.isLength(value, ABUSES_CONSTRAINTS_FIELDS.REASON)
12}
13
14function isAbusePredefinedReasonValid (value: AbusePredefinedReasonsString) {
15 return exists(value) && value in abusePredefinedReasonsMap
16}
17
18function isAbuseFilterValid (value: AbuseFilter) {
19 return value === 'video' || value === 'comment' || value === 'account'
20}
21
22function areAbusePredefinedReasonsValid (value: AbusePredefinedReasonsString[]) {
23 return exists(value) && isArray(value) && value.every(v => v in abusePredefinedReasonsMap)
24}
25
26function isAbuseTimestampValid (value: number) {
27 return value === null || (exists(value) && validator.isInt('' + value, { min: 0 }))
28}
29
30function isAbuseTimestampCoherent (endAt: number, { req }) {
31 const startAt = (req.body as AbuseCreate).video.startAt
32
33 return exists(startAt) && endAt > startAt
34}
35
36function isAbuseModerationCommentValid (value: string) {
37 return exists(value) && validator.isLength(value, ABUSES_CONSTRAINTS_FIELDS.MODERATION_COMMENT)
38}
39
40function isAbuseStateValid (value: string) {
41 return exists(value) && ABUSE_STATES[value] !== undefined
42}
43
44function isAbuseVideoIsValid (value: AbuseVideoIs) {
45 return exists(value) && (
46 value === 'deleted' ||
47 value === 'blacklisted'
48 )
49}
50
51function isAbuseMessageValid (value: string) {
52 return exists(value) && validator.isLength(value, ABUSE_MESSAGES_CONSTRAINTS_FIELDS.MESSAGE)
53}
54
55// ---------------------------------------------------------------------------
56
57export {
58 isAbuseReasonValid,
59 isAbuseFilterValid,
60 isAbusePredefinedReasonValid,
61 isAbuseMessageValid,
62 areAbusePredefinedReasonsValid,
63 isAbuseTimestampValid,
64 isAbuseTimestampCoherent,
65 isAbuseModerationCommentValid,
66 isAbuseStateValid,
67 isAbuseVideoIsValid
68}
diff --git a/server/helpers/custom-validators/accounts.ts b/server/helpers/custom-validators/accounts.ts
deleted file mode 100644
index f676669ea..000000000
--- a/server/helpers/custom-validators/accounts.ts
+++ /dev/null
@@ -1,22 +0,0 @@
1import { isUserDescriptionValid, isUserUsernameValid } from './users'
2import { exists } from './misc'
3
4function isAccountNameValid (value: string) {
5 return isUserUsernameValid(value)
6}
7
8function isAccountIdValid (value: string) {
9 return exists(value)
10}
11
12function isAccountDescriptionValid (value: string) {
13 return isUserDescriptionValid(value)
14}
15
16// ---------------------------------------------------------------------------
17
18export {
19 isAccountIdValid,
20 isAccountDescriptionValid,
21 isAccountNameValid
22}
diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts
deleted file mode 100644
index 90a918523..000000000
--- a/server/helpers/custom-validators/activitypub/activity.ts
+++ /dev/null
@@ -1,151 +0,0 @@
1import validator from 'validator'
2import { Activity, ActivityType } from '../../../../shared/models/activitypub'
3import { isAbuseReasonValid } from '../abuses'
4import { exists } from '../misc'
5import { sanitizeAndCheckActorObject } from './actor'
6import { isCacheFileObjectValid } from './cache-file'
7import { isActivityPubUrlValid, isBaseActivityValid, isObjectValid } from './misc'
8import { isPlaylistObjectValid } from './playlist'
9import { sanitizeAndCheckVideoCommentObject } from './video-comments'
10import { sanitizeAndCheckVideoTorrentObject } from './videos'
11import { isWatchActionObjectValid } from './watch-action'
12
13function isRootActivityValid (activity: any) {
14 return isCollection(activity) || isActivity(activity)
15}
16
17function isCollection (activity: any) {
18 return (activity.type === 'Collection' || activity.type === 'OrderedCollection') &&
19 validator.isInt(activity.totalItems, { min: 0 }) &&
20 Array.isArray(activity.items)
21}
22
23function isActivity (activity: any) {
24 return isActivityPubUrlValid(activity.id) &&
25 exists(activity.actor) &&
26 (isActivityPubUrlValid(activity.actor) || isActivityPubUrlValid(activity.actor.id))
27}
28
29const activityCheckers: { [ P in ActivityType ]: (activity: Activity) => boolean } = {
30 Create: isCreateActivityValid,
31 Update: isUpdateActivityValid,
32 Delete: isDeleteActivityValid,
33 Follow: isFollowActivityValid,
34 Accept: isAcceptActivityValid,
35 Reject: isRejectActivityValid,
36 Announce: isAnnounceActivityValid,
37 Undo: isUndoActivityValid,
38 Like: isLikeActivityValid,
39 View: isViewActivityValid,
40 Flag: isFlagActivityValid,
41 Dislike: isDislikeActivityValid
42}
43
44function isActivityValid (activity: any) {
45 const checker = activityCheckers[activity.type]
46 // Unknown activity type
47 if (!checker) return false
48
49 return checker(activity)
50}
51
52function isFlagActivityValid (activity: any) {
53 return isBaseActivityValid(activity, 'Flag') &&
54 isAbuseReasonValid(activity.content) &&
55 isActivityPubUrlValid(activity.object)
56}
57
58function isLikeActivityValid (activity: any) {
59 return isBaseActivityValid(activity, 'Like') &&
60 isObjectValid(activity.object)
61}
62
63function isDislikeActivityValid (activity: any) {
64 return isBaseActivityValid(activity, 'Dislike') &&
65 isObjectValid(activity.object)
66}
67
68function isAnnounceActivityValid (activity: any) {
69 return isBaseActivityValid(activity, 'Announce') &&
70 isObjectValid(activity.object)
71}
72
73function isViewActivityValid (activity: any) {
74 return isBaseActivityValid(activity, 'View') &&
75 isActivityPubUrlValid(activity.actor) &&
76 isActivityPubUrlValid(activity.object)
77}
78
79function isCreateActivityValid (activity: any) {
80 return isBaseActivityValid(activity, 'Create') &&
81 (
82 isViewActivityValid(activity.object) ||
83 isDislikeActivityValid(activity.object) ||
84 isFlagActivityValid(activity.object) ||
85 isPlaylistObjectValid(activity.object) ||
86 isWatchActionObjectValid(activity.object) ||
87
88 isCacheFileObjectValid(activity.object) ||
89 sanitizeAndCheckVideoCommentObject(activity.object) ||
90 sanitizeAndCheckVideoTorrentObject(activity.object)
91 )
92}
93
94function isUpdateActivityValid (activity: any) {
95 return isBaseActivityValid(activity, 'Update') &&
96 (
97 isCacheFileObjectValid(activity.object) ||
98 isPlaylistObjectValid(activity.object) ||
99 sanitizeAndCheckVideoTorrentObject(activity.object) ||
100 sanitizeAndCheckActorObject(activity.object)
101 )
102}
103
104function isDeleteActivityValid (activity: any) {
105 // We don't really check objects
106 return isBaseActivityValid(activity, 'Delete') &&
107 isObjectValid(activity.object)
108}
109
110function isFollowActivityValid (activity: any) {
111 return isBaseActivityValid(activity, 'Follow') &&
112 isObjectValid(activity.object)
113}
114
115function isAcceptActivityValid (activity: any) {
116 return isBaseActivityValid(activity, 'Accept')
117}
118
119function isRejectActivityValid (activity: any) {
120 return isBaseActivityValid(activity, 'Reject')
121}
122
123function isUndoActivityValid (activity: any) {
124 return isBaseActivityValid(activity, 'Undo') &&
125 (
126 isFollowActivityValid(activity.object) ||
127 isLikeActivityValid(activity.object) ||
128 isDislikeActivityValid(activity.object) ||
129 isAnnounceActivityValid(activity.object) ||
130 isCreateActivityValid(activity.object)
131 )
132}
133
134// ---------------------------------------------------------------------------
135
136export {
137 isRootActivityValid,
138 isActivityValid,
139 isFlagActivityValid,
140 isLikeActivityValid,
141 isDislikeActivityValid,
142 isAnnounceActivityValid,
143 isViewActivityValid,
144 isCreateActivityValid,
145 isUpdateActivityValid,
146 isDeleteActivityValid,
147 isFollowActivityValid,
148 isAcceptActivityValid,
149 isRejectActivityValid,
150 isUndoActivityValid
151}
diff --git a/server/helpers/custom-validators/activitypub/actor.ts b/server/helpers/custom-validators/activitypub/actor.ts
deleted file mode 100644
index f43c35b23..000000000
--- a/server/helpers/custom-validators/activitypub/actor.ts
+++ /dev/null
@@ -1,142 +0,0 @@
1import validator from 'validator'
2import { CONSTRAINTS_FIELDS } from '../../../initializers/constants'
3import { exists, isArray, isDateValid } from '../misc'
4import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc'
5import { isHostValid } from '../servers'
6import { peertubeTruncate } from '@server/helpers/core-utils'
7
8function isActorEndpointsObjectValid (endpointObject: any) {
9 if (endpointObject?.sharedInbox) {
10 return isActivityPubUrlValid(endpointObject.sharedInbox)
11 }
12
13 // Shared inbox is optional
14 return true
15}
16
17function isActorPublicKeyObjectValid (publicKeyObject: any) {
18 return isActivityPubUrlValid(publicKeyObject.id) &&
19 isActivityPubUrlValid(publicKeyObject.owner) &&
20 isActorPublicKeyValid(publicKeyObject.publicKeyPem)
21}
22
23function isActorTypeValid (type: string) {
24 return type === 'Person' || type === 'Application' || type === 'Group' || type === 'Service' || type === 'Organization'
25}
26
27function isActorPublicKeyValid (publicKey: string) {
28 return exists(publicKey) &&
29 typeof publicKey === 'string' &&
30 publicKey.startsWith('-----BEGIN PUBLIC KEY-----') &&
31 publicKey.includes('-----END PUBLIC KEY-----') &&
32 validator.isLength(publicKey, CONSTRAINTS_FIELDS.ACTORS.PUBLIC_KEY)
33}
34
35const actorNameAlphabet = '[ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789\\-_.:]'
36const actorNameRegExp = new RegExp(`^${actorNameAlphabet}+$`)
37function isActorPreferredUsernameValid (preferredUsername: string) {
38 return exists(preferredUsername) && validator.matches(preferredUsername, actorNameRegExp)
39}
40
41function isActorPrivateKeyValid (privateKey: string) {
42 return exists(privateKey) &&
43 typeof privateKey === 'string' &&
44 (privateKey.startsWith('-----BEGIN RSA PRIVATE KEY-----') || privateKey.startsWith('-----BEGIN PRIVATE KEY-----')) &&
45 // Sometimes there is a \n at the end, so just assert the string contains the end mark
46 (privateKey.includes('-----END RSA PRIVATE KEY-----') || privateKey.includes('-----END PRIVATE KEY-----')) &&
47 validator.isLength(privateKey, CONSTRAINTS_FIELDS.ACTORS.PRIVATE_KEY)
48}
49
50function isActorFollowingCountValid (value: string) {
51 return exists(value) && validator.isInt('' + value, { min: 0 })
52}
53
54function isActorFollowersCountValid (value: string) {
55 return exists(value) && validator.isInt('' + value, { min: 0 })
56}
57
58function isActorDeleteActivityValid (activity: any) {
59 return isBaseActivityValid(activity, 'Delete')
60}
61
62function sanitizeAndCheckActorObject (actor: any) {
63 if (!isActorTypeValid(actor.type)) return false
64
65 normalizeActor(actor)
66
67 return exists(actor) &&
68 isActivityPubUrlValid(actor.id) &&
69 isActivityPubUrlValid(actor.inbox) &&
70 isActorPreferredUsernameValid(actor.preferredUsername) &&
71 isActivityPubUrlValid(actor.url) &&
72 isActorPublicKeyObjectValid(actor.publicKey) &&
73 isActorEndpointsObjectValid(actor.endpoints) &&
74
75 (!actor.outbox || isActivityPubUrlValid(actor.outbox)) &&
76 (!actor.following || isActivityPubUrlValid(actor.following)) &&
77 (!actor.followers || isActivityPubUrlValid(actor.followers)) &&
78
79 setValidAttributedTo(actor) &&
80 setValidDescription(actor) &&
81 // If this is a group (a channel), it should be attributed to an account
82 // In PeerTube we use this to attach a video channel to a specific account
83 (actor.type !== 'Group' || actor.attributedTo.length !== 0)
84}
85
86function normalizeActor (actor: any) {
87 if (!actor) return
88
89 if (!actor.url) {
90 actor.url = actor.id
91 } else if (typeof actor.url !== 'string') {
92 actor.url = actor.url.href || actor.url.url
93 }
94
95 if (!isDateValid(actor.published)) actor.published = undefined
96
97 if (actor.summary && typeof actor.summary === 'string') {
98 actor.summary = peertubeTruncate(actor.summary, { length: CONSTRAINTS_FIELDS.USERS.DESCRIPTION.max })
99
100 if (actor.summary.length < CONSTRAINTS_FIELDS.USERS.DESCRIPTION.min) {
101 actor.summary = null
102 }
103 }
104}
105
106function isValidActorHandle (handle: string) {
107 if (!exists(handle)) return false
108
109 const parts = handle.split('@')
110 if (parts.length !== 2) return false
111
112 return isHostValid(parts[1])
113}
114
115function areValidActorHandles (handles: string[]) {
116 return isArray(handles) && handles.every(h => isValidActorHandle(h))
117}
118
119function setValidDescription (obj: any) {
120 if (!obj.summary) obj.summary = null
121
122 return true
123}
124
125// ---------------------------------------------------------------------------
126
127export {
128 normalizeActor,
129 actorNameAlphabet,
130 areValidActorHandles,
131 isActorEndpointsObjectValid,
132 isActorPublicKeyObjectValid,
133 isActorTypeValid,
134 isActorPublicKeyValid,
135 isActorPreferredUsernameValid,
136 isActorPrivateKeyValid,
137 isActorFollowingCountValid,
138 isActorFollowersCountValid,
139 isActorDeleteActivityValid,
140 sanitizeAndCheckActorObject,
141 isValidActorHandle
142}
diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts
deleted file mode 100644
index c5b3b4d9f..000000000
--- a/server/helpers/custom-validators/activitypub/cache-file.ts
+++ /dev/null
@@ -1,26 +0,0 @@
1import { isActivityPubUrlValid } from './misc'
2import { isRemoteVideoUrlValid } from './videos'
3import { exists, isDateValid } from '../misc'
4import { CacheFileObject } from '../../../../shared/models/activitypub/objects'
5
6function isCacheFileObjectValid (object: CacheFileObject) {
7 return exists(object) &&
8 object.type === 'CacheFile' &&
9 (object.expires === null || isDateValid(object.expires)) &&
10 isActivityPubUrlValid(object.object) &&
11 (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url))
12}
13
14// ---------------------------------------------------------------------------
15
16export {
17 isCacheFileObjectValid
18}
19
20// ---------------------------------------------------------------------------
21
22function isPlaylistRedundancyUrlValid (url: any) {
23 return url.type === 'Link' &&
24 (url.mediaType || url.mimeType) === 'application/x-mpegURL' &&
25 isActivityPubUrlValid(url.href)
26}
diff --git a/server/helpers/custom-validators/activitypub/misc.ts b/server/helpers/custom-validators/activitypub/misc.ts
deleted file mode 100644
index 7df47cf15..000000000
--- a/server/helpers/custom-validators/activitypub/misc.ts
+++ /dev/null
@@ -1,76 +0,0 @@
1import validator from 'validator'
2import { CONFIG } from '@server/initializers/config'
3import { CONSTRAINTS_FIELDS } from '../../../initializers/constants'
4import { exists } from '../misc'
5
6function isUrlValid (url: string) {
7 const isURLOptions = {
8 require_host: true,
9 require_tld: true,
10 require_protocol: true,
11 require_valid_protocol: true,
12 protocols: [ 'http', 'https' ]
13 }
14
15 // We validate 'localhost', so we don't have the top level domain
16 if (CONFIG.WEBSERVER.HOSTNAME === 'localhost' || CONFIG.WEBSERVER.HOSTNAME === '127.0.0.1') {
17 isURLOptions.require_tld = false
18 }
19
20 return exists(url) && validator.isURL('' + url, isURLOptions)
21}
22
23function isActivityPubUrlValid (url: string) {
24 return isUrlValid(url) && validator.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL)
25}
26
27function isBaseActivityValid (activity: any, type: string) {
28 return activity.type === type &&
29 isActivityPubUrlValid(activity.id) &&
30 isObjectValid(activity.actor) &&
31 isUrlCollectionValid(activity.to) &&
32 isUrlCollectionValid(activity.cc)
33}
34
35function isUrlCollectionValid (collection: any) {
36 return collection === undefined ||
37 (Array.isArray(collection) && collection.every(t => isActivityPubUrlValid(t)))
38}
39
40function isObjectValid (object: any) {
41 return exists(object) &&
42 (
43 isActivityPubUrlValid(object) || isActivityPubUrlValid(object.id)
44 )
45}
46
47function setValidAttributedTo (obj: any) {
48 if (Array.isArray(obj.attributedTo) === false) {
49 obj.attributedTo = []
50 return true
51 }
52
53 obj.attributedTo = obj.attributedTo.filter(a => {
54 return isActivityPubUrlValid(a) ||
55 ((a.type === 'Group' || a.type === 'Person') && isActivityPubUrlValid(a.id))
56 })
57
58 return true
59}
60
61function isActivityPubVideoDurationValid (value: string) {
62 // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration
63 return exists(value) &&
64 typeof value === 'string' &&
65 value.startsWith('PT') &&
66 value.endsWith('S')
67}
68
69export {
70 isUrlValid,
71 isActivityPubUrlValid,
72 isBaseActivityValid,
73 setValidAttributedTo,
74 isObjectValid,
75 isActivityPubVideoDurationValid
76}
diff --git a/server/helpers/custom-validators/activitypub/playlist.ts b/server/helpers/custom-validators/activitypub/playlist.ts
deleted file mode 100644
index 49bcadcfd..000000000
--- a/server/helpers/custom-validators/activitypub/playlist.ts
+++ /dev/null
@@ -1,29 +0,0 @@
1import validator from 'validator'
2import { PlaylistElementObject, PlaylistObject } from '@shared/models'
3import { exists, isDateValid, isUUIDValid } from '../misc'
4import { isVideoPlaylistNameValid } from '../video-playlists'
5import { isActivityPubUrlValid } from './misc'
6
7function isPlaylistObjectValid (object: PlaylistObject) {
8 return exists(object) &&
9 object.type === 'Playlist' &&
10 validator.isInt(object.totalItems + '') &&
11 isVideoPlaylistNameValid(object.name) &&
12 isUUIDValid(object.uuid) &&
13 isDateValid(object.published) &&
14 isDateValid(object.updated)
15}
16
17function isPlaylistElementObjectValid (object: PlaylistElementObject) {
18 return exists(object) &&
19 object.type === 'PlaylistElement' &&
20 validator.isInt(object.position + '') &&
21 isActivityPubUrlValid(object.url)
22}
23
24// ---------------------------------------------------------------------------
25
26export {
27 isPlaylistObjectValid,
28 isPlaylistElementObjectValid
29}
diff --git a/server/helpers/custom-validators/activitypub/signature.ts b/server/helpers/custom-validators/activitypub/signature.ts
deleted file mode 100644
index cfb65361e..000000000
--- a/server/helpers/custom-validators/activitypub/signature.ts
+++ /dev/null
@@ -1,22 +0,0 @@
1import { exists } from '../misc'
2import { isActivityPubUrlValid } from './misc'
3
4function isSignatureTypeValid (signatureType: string) {
5 return exists(signatureType) && signatureType === 'RsaSignature2017'
6}
7
8function isSignatureCreatorValid (signatureCreator: string) {
9 return exists(signatureCreator) && isActivityPubUrlValid(signatureCreator)
10}
11
12function isSignatureValueValid (signatureValue: string) {
13 return exists(signatureValue) && signatureValue.length > 0
14}
15
16// ---------------------------------------------------------------------------
17
18export {
19 isSignatureTypeValid,
20 isSignatureCreatorValid,
21 isSignatureValueValid
22}
diff --git a/server/helpers/custom-validators/activitypub/video-comments.ts b/server/helpers/custom-validators/activitypub/video-comments.ts
deleted file mode 100644
index ea852c491..000000000
--- a/server/helpers/custom-validators/activitypub/video-comments.ts
+++ /dev/null
@@ -1,59 +0,0 @@
1import validator from 'validator'
2import { ACTIVITY_PUB } from '../../../initializers/constants'
3import { exists, isArray, isDateValid } from '../misc'
4import { isActivityPubUrlValid } from './misc'
5
6function sanitizeAndCheckVideoCommentObject (comment: any) {
7 if (!comment) return false
8
9 if (!isCommentTypeValid(comment)) return false
10
11 normalizeComment(comment)
12
13 if (comment.type === 'Tombstone') {
14 return isActivityPubUrlValid(comment.id) &&
15 isDateValid(comment.published) &&
16 isDateValid(comment.deleted) &&
17 isActivityPubUrlValid(comment.url)
18 }
19
20 return isActivityPubUrlValid(comment.id) &&
21 isCommentContentValid(comment.content) &&
22 isActivityPubUrlValid(comment.inReplyTo) &&
23 isDateValid(comment.published) &&
24 isActivityPubUrlValid(comment.url) &&
25 isArray(comment.to) &&
26 (
27 comment.to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ||
28 comment.cc.indexOf(ACTIVITY_PUB.PUBLIC) !== -1
29 ) // Only accept public comments
30}
31
32// ---------------------------------------------------------------------------
33
34export {
35 sanitizeAndCheckVideoCommentObject
36}
37
38// ---------------------------------------------------------------------------
39
40function isCommentContentValid (content: any) {
41 return exists(content) && validator.isLength('' + content, { min: 1 })
42}
43
44function normalizeComment (comment: any) {
45 if (!comment) return
46
47 if (typeof comment.url !== 'string') {
48 if (typeof comment.url === 'object') comment.url = comment.url.href || comment.url.url
49 else comment.url = comment.id
50 }
51}
52
53function isCommentTypeValid (comment: any): boolean {
54 if (comment.type === 'Note') return true
55
56 if (comment.type === 'Tombstone' && comment.formerType === 'Note') return true
57
58 return false
59}
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
deleted file mode 100644
index 07e25b8ba..000000000
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ /dev/null
@@ -1,241 +0,0 @@
1import validator from 'validator'
2import { logger } from '@server/helpers/logger'
3import { ActivityPubStoryboard, ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject, VideoObject } from '@shared/models'
4import { LiveVideoLatencyMode, VideoState } from '../../../../shared/models/videos'
5import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers/constants'
6import { peertubeTruncate } from '../../core-utils'
7import { isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc'
8import { isLiveLatencyModeValid } from '../video-lives'
9import {
10 isVideoDescriptionValid,
11 isVideoDurationValid,
12 isVideoNameValid,
13 isVideoStateValid,
14 isVideoTagValid,
15 isVideoViewsValid
16} from '../videos'
17import { isActivityPubUrlValid, isActivityPubVideoDurationValid, isBaseActivityValid, setValidAttributedTo } from './misc'
18
19function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) {
20 return isBaseActivityValid(activity, 'Update') &&
21 sanitizeAndCheckVideoTorrentObject(activity.object)
22}
23
24function sanitizeAndCheckVideoTorrentObject (video: any) {
25 if (!video || video.type !== 'Video') return false
26
27 if (!setValidRemoteTags(video)) {
28 logger.debug('Video has invalid tags', { video })
29 return false
30 }
31 if (!setValidRemoteVideoUrls(video)) {
32 logger.debug('Video has invalid urls', { video })
33 return false
34 }
35 if (!setRemoteVideoContent(video)) {
36 logger.debug('Video has invalid content', { video })
37 return false
38 }
39 if (!setValidAttributedTo(video)) {
40 logger.debug('Video has invalid attributedTo', { video })
41 return false
42 }
43 if (!setValidRemoteCaptions(video)) {
44 logger.debug('Video has invalid captions', { video })
45 return false
46 }
47 if (!setValidRemoteIcon(video)) {
48 logger.debug('Video has invalid icons', { video })
49 return false
50 }
51 if (!setValidStoryboard(video)) {
52 logger.debug('Video has invalid preview (storyboard)', { video })
53 return false
54 }
55
56 // Default attributes
57 if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED
58 if (!isBooleanValid(video.waitTranscoding)) video.waitTranscoding = false
59 if (!isBooleanValid(video.downloadEnabled)) video.downloadEnabled = true
60 if (!isBooleanValid(video.commentsEnabled)) video.commentsEnabled = false
61 if (!isBooleanValid(video.isLiveBroadcast)) video.isLiveBroadcast = false
62 if (!isBooleanValid(video.liveSaveReplay)) video.liveSaveReplay = false
63 if (!isBooleanValid(video.permanentLive)) video.permanentLive = false
64 if (!isLiveLatencyModeValid(video.latencyMode)) video.latencyMode = LiveVideoLatencyMode.DEFAULT
65
66 return isActivityPubUrlValid(video.id) &&
67 isVideoNameValid(video.name) &&
68 isActivityPubVideoDurationValid(video.duration) &&
69 isVideoDurationValid(video.duration.replace(/[^0-9]+/g, '')) &&
70 isUUIDValid(video.uuid) &&
71 (!video.category || isRemoteNumberIdentifierValid(video.category)) &&
72 (!video.licence || isRemoteNumberIdentifierValid(video.licence)) &&
73 (!video.language || isRemoteStringIdentifierValid(video.language)) &&
74 isVideoViewsValid(video.views) &&
75 isBooleanValid(video.sensitive) &&
76 isDateValid(video.published) &&
77 isDateValid(video.updated) &&
78 (!video.originallyPublishedAt || isDateValid(video.originallyPublishedAt)) &&
79 (!video.uploadDate || isDateValid(video.uploadDate)) &&
80 (!video.content || isRemoteVideoContentValid(video.mediaType, video.content)) &&
81 video.attributedTo.length !== 0
82}
83
84function isRemoteVideoUrlValid (url: any) {
85 return url.type === 'Link' &&
86 // Video file link
87 (
88 ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.includes(url.mediaType) &&
89 isActivityPubUrlValid(url.href) &&
90 validator.isInt(url.height + '', { min: 0 }) &&
91 validator.isInt(url.size + '', { min: 0 }) &&
92 (!url.fps || validator.isInt(url.fps + '', { min: -1 }))
93 ) ||
94 // Torrent link
95 (
96 ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.includes(url.mediaType) &&
97 isActivityPubUrlValid(url.href) &&
98 validator.isInt(url.height + '', { min: 0 })
99 ) ||
100 // Magnet link
101 (
102 ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.includes(url.mediaType) &&
103 validator.isLength(url.href, { min: 5 }) &&
104 validator.isInt(url.height + '', { min: 0 })
105 ) ||
106 // HLS playlist link
107 (
108 (url.mediaType || url.mimeType) === 'application/x-mpegURL' &&
109 isActivityPubUrlValid(url.href) &&
110 isArray(url.tag)
111 ) ||
112 isAPVideoTrackerUrlObject(url) ||
113 isAPVideoFileUrlMetadataObject(url)
114}
115
116function isAPVideoFileUrlMetadataObject (url: any): url is ActivityVideoFileMetadataUrlObject {
117 return url &&
118 url.type === 'Link' &&
119 url.mediaType === 'application/json' &&
120 isArray(url.rel) && url.rel.includes('metadata')
121}
122
123function isAPVideoTrackerUrlObject (url: any): url is ActivityTrackerUrlObject {
124 return isArray(url.rel) &&
125 url.rel.includes('tracker') &&
126 isActivityPubUrlValid(url.href)
127}
128
129// ---------------------------------------------------------------------------
130
131export {
132 sanitizeAndCheckVideoTorrentUpdateActivity,
133 isRemoteStringIdentifierValid,
134 sanitizeAndCheckVideoTorrentObject,
135 isRemoteVideoUrlValid,
136 isAPVideoFileUrlMetadataObject,
137 isAPVideoTrackerUrlObject
138}
139
140// ---------------------------------------------------------------------------
141
142function setValidRemoteTags (video: any) {
143 if (Array.isArray(video.tag) === false) return false
144
145 video.tag = video.tag.filter(t => {
146 return t.type === 'Hashtag' &&
147 isVideoTagValid(t.name)
148 })
149
150 return true
151}
152
153function setValidRemoteCaptions (video: any) {
154 if (!video.subtitleLanguage) video.subtitleLanguage = []
155
156 if (Array.isArray(video.subtitleLanguage) === false) return false
157
158 video.subtitleLanguage = video.subtitleLanguage.filter(caption => {
159 if (!isActivityPubUrlValid(caption.url)) caption.url = null
160
161 return isRemoteStringIdentifierValid(caption)
162 })
163
164 return true
165}
166
167function isRemoteNumberIdentifierValid (data: any) {
168 return validator.isInt(data.identifier, { min: 0 })
169}
170
171function isRemoteStringIdentifierValid (data: any) {
172 return typeof data.identifier === 'string'
173}
174
175function isRemoteVideoContentValid (mediaType: string, content: string) {
176 return mediaType === 'text/markdown' && isVideoDescriptionValid(content)
177}
178
179function setValidRemoteIcon (video: any) {
180 if (video.icon && !isArray(video.icon)) video.icon = [ video.icon ]
181 if (!video.icon) video.icon = []
182
183 video.icon = video.icon.filter(icon => {
184 return icon.type === 'Image' &&
185 isActivityPubUrlValid(icon.url) &&
186 icon.mediaType === 'image/jpeg' &&
187 validator.isInt(icon.width + '', { min: 0 }) &&
188 validator.isInt(icon.height + '', { min: 0 })
189 })
190
191 return video.icon.length !== 0
192}
193
194function setValidRemoteVideoUrls (video: any) {
195 if (Array.isArray(video.url) === false) return false
196
197 video.url = video.url.filter(u => isRemoteVideoUrlValid(u))
198
199 return true
200}
201
202function setRemoteVideoContent (video: any) {
203 if (video.content) {
204 video.content = peertubeTruncate(video.content, { length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max })
205 }
206
207 return true
208}
209
210function setValidStoryboard (video: VideoObject) {
211 if (!video.preview) return true
212 if (!Array.isArray(video.preview)) return false
213
214 video.preview = video.preview.filter(p => isStorybordValid(p))
215
216 return true
217}
218
219function isStorybordValid (preview: ActivityPubStoryboard) {
220 if (!preview) return false
221
222 if (
223 preview.type !== 'Image' ||
224 !isArray(preview.rel) ||
225 !preview.rel.includes('storyboard')
226 ) {
227 return false
228 }
229
230 preview.url = preview.url.filter(u => {
231 return u.mediaType === 'image/jpeg' &&
232 isActivityPubUrlValid(u.href) &&
233 validator.isInt(u.width + '', { min: 0 }) &&
234 validator.isInt(u.height + '', { min: 0 }) &&
235 validator.isInt(u.tileWidth + '', { min: 0 }) &&
236 validator.isInt(u.tileHeight + '', { min: 0 }) &&
237 isActivityPubVideoDurationValid(u.tileDuration)
238 })
239
240 return preview.url.length !== 0
241}
diff --git a/server/helpers/custom-validators/activitypub/watch-action.ts b/server/helpers/custom-validators/activitypub/watch-action.ts
deleted file mode 100644
index b9ffa63f6..000000000
--- a/server/helpers/custom-validators/activitypub/watch-action.ts
+++ /dev/null
@@ -1,37 +0,0 @@
1import { WatchActionObject } from '@shared/models'
2import { exists, isDateValid, isUUIDValid } from '../misc'
3import { isVideoTimeValid } from '../video-view'
4import { isActivityPubVideoDurationValid, isObjectValid } from './misc'
5
6function isWatchActionObjectValid (action: WatchActionObject) {
7 return exists(action) &&
8 action.type === 'WatchAction' &&
9 isObjectValid(action.id) &&
10 isActivityPubVideoDurationValid(action.duration) &&
11 isDateValid(action.startTime) &&
12 isDateValid(action.endTime) &&
13 isLocationValid(action.location) &&
14 isUUIDValid(action.uuid) &&
15 isObjectValid(action.object) &&
16 isWatchSectionsValid(action.watchSections)
17}
18
19// ---------------------------------------------------------------------------
20
21export {
22 isWatchActionObjectValid
23}
24
25// ---------------------------------------------------------------------------
26
27function isLocationValid (location: any) {
28 if (!location) return true
29
30 return typeof location === 'object' && typeof location.addressCountry === 'string'
31}
32
33function isWatchSectionsValid (sections: WatchActionObject['watchSections']) {
34 return Array.isArray(sections) && sections.every(s => {
35 return isVideoTimeValid(s.startTimestamp) && isVideoTimeValid(s.endTimestamp)
36 })
37}
diff --git a/server/helpers/custom-validators/actor-images.ts b/server/helpers/custom-validators/actor-images.ts
deleted file mode 100644
index 89f5a2262..000000000
--- a/server/helpers/custom-validators/actor-images.ts
+++ /dev/null
@@ -1,24 +0,0 @@
1
2import { UploadFilesForCheck } from 'express'
3import { CONSTRAINTS_FIELDS } from '../../initializers/constants'
4import { isFileValid } from './misc'
5
6const imageMimeTypes = CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME
7 .map(v => v.replace('.', ''))
8 .join('|')
9const imageMimeTypesRegex = `image/(${imageMimeTypes})`
10
11function isActorImageFile (files: UploadFilesForCheck, fieldname: string) {
12 return isFileValid({
13 files,
14 mimeTypeRegex: imageMimeTypesRegex,
15 field: fieldname,
16 maxSize: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max
17 })
18}
19
20// ---------------------------------------------------------------------------
21
22export {
23 isActorImageFile
24}
diff --git a/server/helpers/custom-validators/bulk.ts b/server/helpers/custom-validators/bulk.ts
deleted file mode 100644
index 9e0ce0be1..000000000
--- a/server/helpers/custom-validators/bulk.ts
+++ /dev/null
@@ -1,9 +0,0 @@
1function isBulkRemoveCommentsOfScopeValid (value: string) {
2 return value === 'my-videos' || value === 'instance'
3}
4
5// ---------------------------------------------------------------------------
6
7export {
8 isBulkRemoveCommentsOfScopeValid
9}
diff --git a/server/helpers/custom-validators/feeds.ts b/server/helpers/custom-validators/feeds.ts
deleted file mode 100644
index fa35a7da6..000000000
--- a/server/helpers/custom-validators/feeds.ts
+++ /dev/null
@@ -1,23 +0,0 @@
1import { exists } from './misc'
2
3function isValidRSSFeed (value: string) {
4 if (!exists(value)) return false
5
6 const feedExtensions = [
7 'xml',
8 'json',
9 'json1',
10 'rss',
11 'rss2',
12 'atom',
13 'atom1'
14 ]
15
16 return feedExtensions.includes(value)
17}
18
19// ---------------------------------------------------------------------------
20
21export {
22 isValidRSSFeed
23}
diff --git a/server/helpers/custom-validators/follows.ts b/server/helpers/custom-validators/follows.ts
deleted file mode 100644
index 0bec683c1..000000000
--- a/server/helpers/custom-validators/follows.ts
+++ /dev/null
@@ -1,30 +0,0 @@
1import { exists, isArray } from './misc'
2import { FollowState } from '@shared/models'
3
4function isFollowStateValid (value: FollowState) {
5 if (!exists(value)) return false
6
7 return value === 'pending' || value === 'accepted' || value === 'rejected'
8}
9
10function isRemoteHandleValid (value: string) {
11 if (!exists(value)) return false
12 if (typeof value !== 'string') return false
13
14 return value.includes('@')
15}
16
17function isEachUniqueHandleValid (handles: string[]) {
18 return isArray(handles) &&
19 handles.every(handle => {
20 return isRemoteHandleValid(handle) && handles.indexOf(handle) === handles.lastIndexOf(handle)
21 })
22}
23
24// ---------------------------------------------------------------------------
25
26export {
27 isFollowStateValid,
28 isRemoteHandleValid,
29 isEachUniqueHandleValid
30}
diff --git a/server/helpers/custom-validators/jobs.ts b/server/helpers/custom-validators/jobs.ts
deleted file mode 100644
index c168b3e91..000000000
--- a/server/helpers/custom-validators/jobs.ts
+++ /dev/null
@@ -1,21 +0,0 @@
1import { JobState } from '../../../shared/models'
2import { exists } from './misc'
3import { jobTypes } from '@server/lib/job-queue/job-queue'
4
5const jobStates: JobState[] = [ 'active', 'completed', 'failed', 'waiting', 'delayed', 'paused', 'waiting-children' ]
6
7function isValidJobState (value: JobState) {
8 return exists(value) && jobStates.includes(value)
9}
10
11function isValidJobType (value: any) {
12 return exists(value) && jobTypes.includes(value)
13}
14
15// ---------------------------------------------------------------------------
16
17export {
18 jobStates,
19 isValidJobState,
20 isValidJobType
21}
diff --git a/server/helpers/custom-validators/logs.ts b/server/helpers/custom-validators/logs.ts
deleted file mode 100644
index 215dbb0e1..000000000
--- a/server/helpers/custom-validators/logs.ts
+++ /dev/null
@@ -1,42 +0,0 @@
1import validator from 'validator'
2import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
3import { ClientLogLevel, ServerLogLevel } from '@shared/models'
4import { exists } from './misc'
5
6const serverLogLevels = new Set<ServerLogLevel>([ 'debug', 'info', 'warn', 'error' ])
7const clientLogLevels = new Set<ClientLogLevel>([ 'warn', 'error' ])
8
9function isValidLogLevel (value: any) {
10 return exists(value) && serverLogLevels.has(value)
11}
12
13function isValidClientLogMessage (value: any) {
14 return typeof value === 'string' && validator.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_MESSAGE)
15}
16
17function isValidClientLogLevel (value: any) {
18 return exists(value) && clientLogLevels.has(value)
19}
20
21function isValidClientLogStackTrace (value: any) {
22 return typeof value === 'string' && validator.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_STACK_TRACE)
23}
24
25function isValidClientLogMeta (value: any) {
26 return typeof value === 'string' && validator.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_META)
27}
28
29function isValidClientLogUserAgent (value: any) {
30 return typeof value === 'string' && validator.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_USER_AGENT)
31}
32
33// ---------------------------------------------------------------------------
34
35export {
36 isValidLogLevel,
37 isValidClientLogMessage,
38 isValidClientLogStackTrace,
39 isValidClientLogMeta,
40 isValidClientLogLevel,
41 isValidClientLogUserAgent
42}
diff --git a/server/helpers/custom-validators/metrics.ts b/server/helpers/custom-validators/metrics.ts
deleted file mode 100644
index 44a863630..000000000
--- a/server/helpers/custom-validators/metrics.ts
+++ /dev/null
@@ -1,10 +0,0 @@
1function isValidPlayerMode (value: any) {
2 // TODO: remove webtorrent in v7
3 return value === 'webtorrent' || value === 'web-video' || value === 'p2p-media-loader'
4}
5
6// ---------------------------------------------------------------------------
7
8export {
9 isValidPlayerMode
10}
diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts
deleted file mode 100644
index 937ae0632..000000000
--- a/server/helpers/custom-validators/misc.ts
+++ /dev/null
@@ -1,190 +0,0 @@
1import 'multer'
2import { UploadFilesForCheck } from 'express'
3import { sep } from 'path'
4import validator from 'validator'
5import { isShortUUID, shortToUUID } from '@shared/extra-utils'
6
7function exists (value: any) {
8 return value !== undefined && value !== null
9}
10
11function isSafePath (p: string) {
12 return exists(p) &&
13 (p + '').split(sep).every(part => {
14 return [ '..' ].includes(part) === false
15 })
16}
17
18function isSafeFilename (filename: string, extension?: string) {
19 const regex = extension
20 ? new RegExp(`^[a-z0-9-]+\\.${extension}$`)
21 : new RegExp(`^[a-z0-9-]+\\.[a-z0-9]{1,8}$`)
22
23 return typeof filename === 'string' && !!filename.match(regex)
24}
25
26function isSafePeerTubeFilenameWithoutExtension (filename: string) {
27 return filename.match(/^[a-z0-9-]+$/)
28}
29
30function isArray (value: any): value is any[] {
31 return Array.isArray(value)
32}
33
34function isNotEmptyIntArray (value: any) {
35 return Array.isArray(value) && value.every(v => validator.isInt('' + v)) && value.length !== 0
36}
37
38function isNotEmptyStringArray (value: any) {
39 return Array.isArray(value) && value.every(v => typeof v === 'string' && v.length !== 0) && value.length !== 0
40}
41
42function isArrayOf (value: any, validator: (value: any) => boolean) {
43 return isArray(value) && value.every(v => validator(v))
44}
45
46function isDateValid (value: string) {
47 return exists(value) && validator.isISO8601(value)
48}
49
50function isIdValid (value: string) {
51 return exists(value) && validator.isInt('' + value)
52}
53
54function isUUIDValid (value: string) {
55 return exists(value) && validator.isUUID('' + value, 4)
56}
57
58function areUUIDsValid (values: string[]) {
59 return isArray(values) && values.every(v => isUUIDValid(v))
60}
61
62function isIdOrUUIDValid (value: string) {
63 return isIdValid(value) || isUUIDValid(value)
64}
65
66function isBooleanValid (value: any) {
67 return typeof value === 'boolean' || (typeof value === 'string' && validator.isBoolean(value))
68}
69
70function isIntOrNull (value: any) {
71 return value === null || validator.isInt('' + value)
72}
73
74// ---------------------------------------------------------------------------
75
76function isFileValid (options: {
77 files: UploadFilesForCheck
78
79 maxSize: number | null
80 mimeTypeRegex: string | null
81
82 field?: string
83
84 optional?: boolean // Default false
85}) {
86 const { files, mimeTypeRegex, field, maxSize, optional = false } = options
87
88 // Should have files
89 if (!files) return optional
90
91 const fileArray = isArray(files)
92 ? files
93 : files[field]
94
95 if (!fileArray || !isArray(fileArray) || fileArray.length === 0) {
96 return optional
97 }
98
99 // The file exists
100 const file = fileArray[0]
101 if (!file?.originalname) return false
102
103 // Check size
104 if ((maxSize !== null) && file.size > maxSize) return false
105
106 if (mimeTypeRegex === null) return true
107
108 return checkMimetypeRegex(file.mimetype, mimeTypeRegex)
109}
110
111function checkMimetypeRegex (fileMimeType: string, mimeTypeRegex: string) {
112 return new RegExp(`^${mimeTypeRegex}$`, 'i').test(fileMimeType)
113}
114
115// ---------------------------------------------------------------------------
116
117function toCompleteUUID (value: string) {
118 if (isShortUUID(value)) {
119 try {
120 return shortToUUID(value)
121 } catch {
122 return ''
123 }
124 }
125
126 return value
127}
128
129function toCompleteUUIDs (values: string[]) {
130 return values.map(v => toCompleteUUID(v))
131}
132
133function toIntOrNull (value: string) {
134 const v = toValueOrNull(value)
135
136 if (v === null || v === undefined) return v
137 if (typeof v === 'number') return v
138
139 return validator.toInt('' + v)
140}
141
142function toBooleanOrNull (value: any) {
143 const v = toValueOrNull(value)
144
145 if (v === null || v === undefined) return v
146 if (typeof v === 'boolean') return v
147
148 return validator.toBoolean('' + v)
149}
150
151function toValueOrNull (value: string) {
152 if (value === 'null') return null
153
154 return value
155}
156
157function toIntArray (value: any) {
158 if (!value) return []
159 if (isArray(value) === false) return [ validator.toInt(value) ]
160
161 return value.map(v => validator.toInt(v))
162}
163
164// ---------------------------------------------------------------------------
165
166export {
167 exists,
168 isArrayOf,
169 isNotEmptyIntArray,
170 isArray,
171 isIntOrNull,
172 isIdValid,
173 isSafePath,
174 isNotEmptyStringArray,
175 isUUIDValid,
176 toCompleteUUIDs,
177 toCompleteUUID,
178 isIdOrUUIDValid,
179 isDateValid,
180 toValueOrNull,
181 toBooleanOrNull,
182 isBooleanValid,
183 toIntOrNull,
184 areUUIDsValid,
185 toIntArray,
186 isFileValid,
187 isSafePeerTubeFilenameWithoutExtension,
188 isSafeFilename,
189 checkMimetypeRegex
190}
diff --git a/server/helpers/custom-validators/plugins.ts b/server/helpers/custom-validators/plugins.ts
deleted file mode 100644
index a20de0c4a..000000000
--- a/server/helpers/custom-validators/plugins.ts
+++ /dev/null
@@ -1,178 +0,0 @@
1import validator from 'validator'
2import { PluginPackageJSON } from '../../../shared/models/plugins/plugin-package-json.model'
3import { PluginType } from '../../../shared/models/plugins/plugin.type'
4import { CONSTRAINTS_FIELDS } from '../../initializers/constants'
5import { isUrlValid } from './activitypub/misc'
6import { exists, isArray, isSafePath } from './misc'
7
8const PLUGINS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.PLUGINS
9
10function isPluginTypeValid (value: any) {
11 return exists(value) &&
12 (value === PluginType.PLUGIN || value === PluginType.THEME)
13}
14
15function isPluginNameValid (value: string) {
16 return exists(value) &&
17 validator.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.NAME) &&
18 validator.matches(value, /^[a-z-0-9]+$/)
19}
20
21function isNpmPluginNameValid (value: string) {
22 return exists(value) &&
23 validator.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.NAME) &&
24 validator.matches(value, /^[a-z\-._0-9]+$/) &&
25 (value.startsWith('peertube-plugin-') || value.startsWith('peertube-theme-'))
26}
27
28function isPluginDescriptionValid (value: string) {
29 return exists(value) && validator.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.DESCRIPTION)
30}
31
32function isPluginStableVersionValid (value: string) {
33 if (!exists(value)) return false
34
35 const parts = (value + '').split('.')
36
37 return parts.length === 3 && parts.every(p => validator.isInt(p))
38}
39
40function isPluginStableOrUnstableVersionValid (value: string) {
41 if (!exists(value)) return false
42
43 // suffix is beta.x or alpha.x
44 const [ stable, suffix ] = value.split('-')
45 if (!isPluginStableVersionValid(stable)) return false
46
47 const suffixRegex = /^(rc|alpha|beta)\.\d+$/
48 if (suffix && !suffixRegex.test(suffix)) return false
49
50 return true
51}
52
53function isPluginEngineValid (engine: any) {
54 return exists(engine) && exists(engine.peertube)
55}
56
57function isPluginHomepage (value: string) {
58 return exists(value) && (!value || isUrlValid(value))
59}
60
61function isPluginBugs (value: string) {
62 return exists(value) && (!value || isUrlValid(value))
63}
64
65function areStaticDirectoriesValid (staticDirs: any) {
66 if (!exists(staticDirs) || typeof staticDirs !== 'object') return false
67
68 for (const key of Object.keys(staticDirs)) {
69 if (!isSafePath(staticDirs[key])) return false
70 }
71
72 return true
73}
74
75function areClientScriptsValid (clientScripts: any[]) {
76 return isArray(clientScripts) &&
77 clientScripts.every(c => {
78 return isSafePath(c.script) && isArray(c.scopes)
79 })
80}
81
82function areTranslationPathsValid (translations: any) {
83 if (!exists(translations) || typeof translations !== 'object') return false
84
85 for (const key of Object.keys(translations)) {
86 if (!isSafePath(translations[key])) return false
87 }
88
89 return true
90}
91
92function areCSSPathsValid (css: any[]) {
93 return isArray(css) && css.every(c => isSafePath(c))
94}
95
96function isThemeNameValid (name: string) {
97 return isPluginNameValid(name)
98}
99
100function isPackageJSONValid (packageJSON: PluginPackageJSON, pluginType: PluginType) {
101 let result = true
102 const badFields: string[] = []
103
104 if (!isNpmPluginNameValid(packageJSON.name)) {
105 result = false
106 badFields.push('name')
107 }
108
109 if (!isPluginDescriptionValid(packageJSON.description)) {
110 result = false
111 badFields.push('description')
112 }
113
114 if (!isPluginEngineValid(packageJSON.engine)) {
115 result = false
116 badFields.push('engine')
117 }
118
119 if (!isPluginHomepage(packageJSON.homepage)) {
120 result = false
121 badFields.push('homepage')
122 }
123
124 if (!exists(packageJSON.author)) {
125 result = false
126 badFields.push('author')
127 }
128
129 if (!isPluginBugs(packageJSON.bugs)) {
130 result = false
131 badFields.push('bugs')
132 }
133
134 if (pluginType === PluginType.PLUGIN && !isSafePath(packageJSON.library)) {
135 result = false
136 badFields.push('library')
137 }
138
139 if (!areStaticDirectoriesValid(packageJSON.staticDirs)) {
140 result = false
141 badFields.push('staticDirs')
142 }
143
144 if (!areCSSPathsValid(packageJSON.css)) {
145 result = false
146 badFields.push('css')
147 }
148
149 if (!areClientScriptsValid(packageJSON.clientScripts)) {
150 result = false
151 badFields.push('clientScripts')
152 }
153
154 if (!areTranslationPathsValid(packageJSON.translations)) {
155 result = false
156 badFields.push('translations')
157 }
158
159 return { result, badFields }
160}
161
162function isLibraryCodeValid (library: any) {
163 return typeof library.register === 'function' &&
164 typeof library.unregister === 'function'
165}
166
167export {
168 isPluginTypeValid,
169 isPackageJSONValid,
170 isThemeNameValid,
171 isPluginHomepage,
172 isPluginStableVersionValid,
173 isPluginStableOrUnstableVersionValid,
174 isPluginNameValid,
175 isPluginDescriptionValid,
176 isLibraryCodeValid,
177 isNpmPluginNameValid
178}
diff --git a/server/helpers/custom-validators/runners/jobs.ts b/server/helpers/custom-validators/runners/jobs.ts
deleted file mode 100644
index 6349e79ba..000000000
--- a/server/helpers/custom-validators/runners/jobs.ts
+++ /dev/null
@@ -1,197 +0,0 @@
1import { UploadFilesForCheck } from 'express'
2import validator from 'validator'
3import { CONSTRAINTS_FIELDS, RUNNER_JOB_STATES } from '@server/initializers/constants'
4import {
5 LiveRTMPHLSTranscodingSuccess,
6 RunnerJobSuccessPayload,
7 RunnerJobType,
8 RunnerJobUpdatePayload,
9 VideoStudioTranscodingSuccess,
10 VODAudioMergeTranscodingSuccess,
11 VODHLSTranscodingSuccess,
12 VODWebVideoTranscodingSuccess
13} from '@shared/models'
14import { exists, isArray, isFileValid, isSafeFilename } from '../misc'
15
16const RUNNER_JOBS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNER_JOBS
17
18const runnerJobTypes = new Set([ 'vod-hls-transcoding', 'vod-web-video-transcoding', 'vod-audio-merge-transcoding' ])
19function isRunnerJobTypeValid (value: RunnerJobType) {
20 return runnerJobTypes.has(value)
21}
22
23function isRunnerJobSuccessPayloadValid (value: RunnerJobSuccessPayload, type: RunnerJobType, files: UploadFilesForCheck) {
24 return isRunnerJobVODWebVideoResultPayloadValid(value as VODWebVideoTranscodingSuccess, type, files) ||
25 isRunnerJobVODHLSResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) ||
26 isRunnerJobVODAudioMergeResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) ||
27 isRunnerJobLiveRTMPHLSResultPayloadValid(value as LiveRTMPHLSTranscodingSuccess, type) ||
28 isRunnerJobVideoStudioResultPayloadValid(value as VideoStudioTranscodingSuccess, type, files)
29}
30
31// ---------------------------------------------------------------------------
32
33function isRunnerJobProgressValid (value: string) {
34 return validator.isInt(value + '', RUNNER_JOBS_CONSTRAINTS_FIELDS.PROGRESS)
35}
36
37function isRunnerJobUpdatePayloadValid (value: RunnerJobUpdatePayload, type: RunnerJobType, files: UploadFilesForCheck) {
38 return isRunnerJobVODWebVideoUpdatePayloadValid(value, type, files) ||
39 isRunnerJobVODHLSUpdatePayloadValid(value, type, files) ||
40 isRunnerJobVideoStudioUpdatePayloadValid(value, type, files) ||
41 isRunnerJobVODAudioMergeUpdatePayloadValid(value, type, files) ||
42 isRunnerJobLiveRTMPHLSUpdatePayloadValid(value, type, files)
43}
44
45// ---------------------------------------------------------------------------
46
47function isRunnerJobTokenValid (value: string) {
48 return exists(value) && validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.TOKEN)
49}
50
51function isRunnerJobAbortReasonValid (value: string) {
52 return validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.REASON)
53}
54
55function isRunnerJobErrorMessageValid (value: string) {
56 return validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.ERROR_MESSAGE)
57}
58
59function isRunnerJobStateValid (value: any) {
60 return exists(value) && RUNNER_JOB_STATES[value] !== undefined
61}
62
63function isRunnerJobArrayOfStateValid (value: any) {
64 return isArray(value) && value.every(v => isRunnerJobStateValid(v))
65}
66
67// ---------------------------------------------------------------------------
68
69export {
70 isRunnerJobTypeValid,
71 isRunnerJobSuccessPayloadValid,
72 isRunnerJobUpdatePayloadValid,
73 isRunnerJobTokenValid,
74 isRunnerJobErrorMessageValid,
75 isRunnerJobProgressValid,
76 isRunnerJobAbortReasonValid,
77 isRunnerJobArrayOfStateValid,
78 isRunnerJobStateValid
79}
80
81// ---------------------------------------------------------------------------
82
83function isRunnerJobVODWebVideoResultPayloadValid (
84 _value: VODWebVideoTranscodingSuccess,
85 type: RunnerJobType,
86 files: UploadFilesForCheck
87) {
88 return type === 'vod-web-video-transcoding' &&
89 isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null })
90}
91
92function isRunnerJobVODHLSResultPayloadValid (
93 _value: VODHLSTranscodingSuccess,
94 type: RunnerJobType,
95 files: UploadFilesForCheck
96) {
97 return type === 'vod-hls-transcoding' &&
98 isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) &&
99 isFileValid({ files, field: 'payload[resolutionPlaylistFile]', mimeTypeRegex: null, maxSize: null })
100}
101
102function isRunnerJobVODAudioMergeResultPayloadValid (
103 _value: VODAudioMergeTranscodingSuccess,
104 type: RunnerJobType,
105 files: UploadFilesForCheck
106) {
107 return type === 'vod-audio-merge-transcoding' &&
108 isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null })
109}
110
111function isRunnerJobLiveRTMPHLSResultPayloadValid (
112 value: LiveRTMPHLSTranscodingSuccess,
113 type: RunnerJobType
114) {
115 return type === 'live-rtmp-hls-transcoding' && (!value || (typeof value === 'object' && Object.keys(value).length === 0))
116}
117
118function isRunnerJobVideoStudioResultPayloadValid (
119 _value: VideoStudioTranscodingSuccess,
120 type: RunnerJobType,
121 files: UploadFilesForCheck
122) {
123 return type === 'video-studio-transcoding' &&
124 isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null })
125}
126
127// ---------------------------------------------------------------------------
128
129function isRunnerJobVODWebVideoUpdatePayloadValid (
130 value: RunnerJobUpdatePayload,
131 type: RunnerJobType,
132 _files: UploadFilesForCheck
133) {
134 return type === 'vod-web-video-transcoding' &&
135 (!value || (typeof value === 'object' && Object.keys(value).length === 0))
136}
137
138function isRunnerJobVODHLSUpdatePayloadValid (
139 value: RunnerJobUpdatePayload,
140 type: RunnerJobType,
141 _files: UploadFilesForCheck
142) {
143 return type === 'vod-hls-transcoding' &&
144 (!value || (typeof value === 'object' && Object.keys(value).length === 0))
145}
146
147function isRunnerJobVODAudioMergeUpdatePayloadValid (
148 value: RunnerJobUpdatePayload,
149 type: RunnerJobType,
150 _files: UploadFilesForCheck
151) {
152 return type === 'vod-audio-merge-transcoding' &&
153 (!value || (typeof value === 'object' && Object.keys(value).length === 0))
154}
155
156function isRunnerJobLiveRTMPHLSUpdatePayloadValid (
157 value: RunnerJobUpdatePayload,
158 type: RunnerJobType,
159 files: UploadFilesForCheck
160) {
161 let result = type === 'live-rtmp-hls-transcoding' && !!value && !!files
162
163 result &&= isFileValid({ files, field: 'payload[masterPlaylistFile]', mimeTypeRegex: null, maxSize: null, optional: true })
164
165 result &&= isFileValid({
166 files,
167 field: 'payload[resolutionPlaylistFile]',
168 mimeTypeRegex: null,
169 maxSize: null,
170 optional: !value.resolutionPlaylistFilename
171 })
172
173 if (files['payload[resolutionPlaylistFile]']) {
174 result &&= isSafeFilename(value.resolutionPlaylistFilename, 'm3u8')
175 }
176
177 return result &&
178 isSafeFilename(value.videoChunkFilename, 'ts') &&
179 (
180 (
181 value.type === 'remove-chunk'
182 ) ||
183 (
184 value.type === 'add-chunk' &&
185 isFileValid({ files, field: 'payload[videoChunkFile]', mimeTypeRegex: null, maxSize: null })
186 )
187 )
188}
189
190function isRunnerJobVideoStudioUpdatePayloadValid (
191 value: RunnerJobUpdatePayload,
192 type: RunnerJobType,
193 _files: UploadFilesForCheck
194) {
195 return type === 'video-studio-transcoding' &&
196 (!value || (typeof value === 'object' && Object.keys(value).length === 0))
197}
diff --git a/server/helpers/custom-validators/runners/runners.ts b/server/helpers/custom-validators/runners/runners.ts
deleted file mode 100644
index 953fac3b5..000000000
--- a/server/helpers/custom-validators/runners/runners.ts
+++ /dev/null
@@ -1,30 +0,0 @@
1import validator from 'validator'
2import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
3import { exists } from '../misc'
4
5const RUNNERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNERS
6
7function isRunnerRegistrationTokenValid (value: string) {
8 return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.TOKEN)
9}
10
11function isRunnerTokenValid (value: string) {
12 return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.TOKEN)
13}
14
15function isRunnerNameValid (value: string) {
16 return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.NAME)
17}
18
19function isRunnerDescriptionValid (value: string) {
20 return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.DESCRIPTION)
21}
22
23// ---------------------------------------------------------------------------
24
25export {
26 isRunnerRegistrationTokenValid,
27 isRunnerTokenValid,
28 isRunnerNameValid,
29 isRunnerDescriptionValid
30}
diff --git a/server/helpers/custom-validators/search.ts b/server/helpers/custom-validators/search.ts
deleted file mode 100644
index 6dba5d14e..000000000
--- a/server/helpers/custom-validators/search.ts
+++ /dev/null
@@ -1,37 +0,0 @@
1import validator from 'validator'
2import { SearchTargetType } from '@shared/models/search/search-target-query.model'
3import { isArray, exists } from './misc'
4import { CONFIG } from '@server/initializers/config'
5
6function isNumberArray (value: any) {
7 return isArray(value) && value.every(v => validator.isInt('' + v))
8}
9
10function isStringArray (value: any) {
11 return isArray(value) && value.every(v => typeof v === 'string')
12}
13
14function isBooleanBothQueryValid (value: any) {
15 return value === 'true' || value === 'false' || value === 'both'
16}
17
18function isSearchTargetValid (value: SearchTargetType) {
19 if (!exists(value)) return true
20
21 const searchIndexConfig = CONFIG.SEARCH.SEARCH_INDEX
22
23 if (value === 'local') return true
24
25 if (value === 'search-index' && searchIndexConfig.ENABLED) return true
26
27 return false
28}
29
30// ---------------------------------------------------------------------------
31
32export {
33 isNumberArray,
34 isStringArray,
35 isBooleanBothQueryValid,
36 isSearchTargetValid
37}
diff --git a/server/helpers/custom-validators/servers.ts b/server/helpers/custom-validators/servers.ts
deleted file mode 100644
index b2aa03b77..000000000
--- a/server/helpers/custom-validators/servers.ts
+++ /dev/null
@@ -1,42 +0,0 @@
1import validator from 'validator'
2import { CONFIG } from '@server/initializers/config'
3import { CONSTRAINTS_FIELDS } from '../../initializers/constants'
4import { exists, isArray } from './misc'
5
6function isHostValid (host: string) {
7 const isURLOptions = {
8 require_host: true,
9 require_tld: true
10 }
11
12 // We validate 'localhost', so we don't have the top level domain
13 if (CONFIG.WEBSERVER.HOSTNAME === 'localhost' || CONFIG.WEBSERVER.HOSTNAME === '127.0.0.1') {
14 isURLOptions.require_tld = false
15 }
16
17 return exists(host) && validator.isURL(host, isURLOptions) && host.split('://').length === 1
18}
19
20function isEachUniqueHostValid (hosts: string[]) {
21 return isArray(hosts) &&
22 hosts.every(host => {
23 return isHostValid(host) && hosts.indexOf(host) === hosts.lastIndexOf(host)
24 })
25}
26
27function isValidContactBody (value: any) {
28 return exists(value) && validator.isLength(value, CONSTRAINTS_FIELDS.CONTACT_FORM.BODY)
29}
30
31function isValidContactFromName (value: any) {
32 return exists(value) && validator.isLength(value, CONSTRAINTS_FIELDS.CONTACT_FORM.FROM_NAME)
33}
34
35// ---------------------------------------------------------------------------
36
37export {
38 isValidContactBody,
39 isValidContactFromName,
40 isEachUniqueHostValid,
41 isHostValid
42}
diff --git a/server/helpers/custom-validators/user-notifications.ts b/server/helpers/custom-validators/user-notifications.ts
deleted file mode 100644
index 2de13ca09..000000000
--- a/server/helpers/custom-validators/user-notifications.ts
+++ /dev/null
@@ -1,23 +0,0 @@
1import validator from 'validator'
2import { UserNotificationSettingValue } from '@shared/models'
3import { exists } from './misc'
4
5function isUserNotificationTypeValid (value: any) {
6 return exists(value) && validator.isInt('' + value)
7}
8
9function isUserNotificationSettingValid (value: any) {
10 return exists(value) &&
11 validator.isInt('' + value) &&
12 (
13 value === UserNotificationSettingValue.NONE ||
14 value === UserNotificationSettingValue.WEB ||
15 value === UserNotificationSettingValue.EMAIL ||
16 value === (UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL)
17 )
18}
19
20export {
21 isUserNotificationSettingValid,
22 isUserNotificationTypeValid
23}
diff --git a/server/helpers/custom-validators/user-registration.ts b/server/helpers/custom-validators/user-registration.ts
deleted file mode 100644
index 9da0bb08a..000000000
--- a/server/helpers/custom-validators/user-registration.ts
+++ /dev/null
@@ -1,25 +0,0 @@
1import validator from 'validator'
2import { CONSTRAINTS_FIELDS, USER_REGISTRATION_STATES } from '../../initializers/constants'
3import { exists } from './misc'
4
5const USER_REGISTRATIONS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.USER_REGISTRATIONS
6
7function isRegistrationStateValid (value: string) {
8 return exists(value) && USER_REGISTRATION_STATES[value] !== undefined
9}
10
11function isRegistrationModerationResponseValid (value: string) {
12 return exists(value) && validator.isLength(value, USER_REGISTRATIONS_CONSTRAINTS_FIELDS.MODERATOR_MESSAGE)
13}
14
15function isRegistrationReasonValid (value: string) {
16 return exists(value) && validator.isLength(value, USER_REGISTRATIONS_CONSTRAINTS_FIELDS.REASON_MESSAGE)
17}
18
19// ---------------------------------------------------------------------------
20
21export {
22 isRegistrationStateValid,
23 isRegistrationModerationResponseValid,
24 isRegistrationReasonValid
25}
diff --git a/server/helpers/custom-validators/users.ts b/server/helpers/custom-validators/users.ts
deleted file mode 100644
index f02b3ba65..000000000
--- a/server/helpers/custom-validators/users.ts
+++ /dev/null
@@ -1,123 +0,0 @@
1import validator from 'validator'
2import { UserRole } from '@shared/models'
3import { isEmailEnabled } from '../../initializers/config'
4import { CONSTRAINTS_FIELDS, NSFW_POLICY_TYPES } from '../../initializers/constants'
5import { exists, isArray, isBooleanValid } from './misc'
6
7const USERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.USERS
8
9function isUserPasswordValid (value: string) {
10 return validator.isLength(value, USERS_CONSTRAINTS_FIELDS.PASSWORD)
11}
12
13function isUserPasswordValidOrEmpty (value: string) {
14 // Empty password is only possible if emailing is enabled.
15 if (value === '') return isEmailEnabled()
16
17 return isUserPasswordValid(value)
18}
19
20function isUserVideoQuotaValid (value: string) {
21 return exists(value) && validator.isInt(value + '', USERS_CONSTRAINTS_FIELDS.VIDEO_QUOTA)
22}
23
24function isUserVideoQuotaDailyValid (value: string) {
25 return exists(value) && validator.isInt(value + '', USERS_CONSTRAINTS_FIELDS.VIDEO_QUOTA_DAILY)
26}
27
28function isUserUsernameValid (value: string) {
29 return exists(value) &&
30 validator.matches(value, new RegExp(`^[a-z0-9_]+([a-z0-9_.-]+[a-z0-9_]+)?$`)) &&
31 validator.isLength(value, USERS_CONSTRAINTS_FIELDS.USERNAME)
32}
33
34function isUserDisplayNameValid (value: string) {
35 return value === null || (exists(value) && validator.isLength(value, CONSTRAINTS_FIELDS.USERS.NAME))
36}
37
38function isUserDescriptionValid (value: string) {
39 return value === null || (exists(value) && validator.isLength(value, CONSTRAINTS_FIELDS.USERS.DESCRIPTION))
40}
41
42function isUserEmailVerifiedValid (value: any) {
43 return isBooleanValid(value)
44}
45
46const nsfwPolicies = new Set(Object.values(NSFW_POLICY_TYPES))
47function isUserNSFWPolicyValid (value: any) {
48 return exists(value) && nsfwPolicies.has(value)
49}
50
51function isUserP2PEnabledValid (value: any) {
52 return isBooleanValid(value)
53}
54
55function isUserVideosHistoryEnabledValid (value: any) {
56 return isBooleanValid(value)
57}
58
59function isUserAutoPlayVideoValid (value: any) {
60 return isBooleanValid(value)
61}
62
63function isUserVideoLanguages (value: any) {
64 return value === null || (isArray(value) && value.length < CONSTRAINTS_FIELDS.USERS.VIDEO_LANGUAGES.max)
65}
66
67function isUserAdminFlagsValid (value: any) {
68 return exists(value) && validator.isInt('' + value)
69}
70
71function isUserBlockedValid (value: any) {
72 return isBooleanValid(value)
73}
74
75function isUserAutoPlayNextVideoValid (value: any) {
76 return isBooleanValid(value)
77}
78
79function isUserAutoPlayNextVideoPlaylistValid (value: any) {
80 return isBooleanValid(value)
81}
82
83function isUserEmailPublicValid (value: any) {
84 return isBooleanValid(value)
85}
86
87function isUserNoModal (value: any) {
88 return isBooleanValid(value)
89}
90
91function isUserBlockedReasonValid (value: any) {
92 return value === null || (exists(value) && validator.isLength(value, CONSTRAINTS_FIELDS.USERS.BLOCKED_REASON))
93}
94
95function isUserRoleValid (value: any) {
96 return exists(value) && validator.isInt('' + value) && [ UserRole.ADMINISTRATOR, UserRole.MODERATOR, UserRole.USER ].includes(value)
97}
98
99// ---------------------------------------------------------------------------
100
101export {
102 isUserVideosHistoryEnabledValid,
103 isUserBlockedValid,
104 isUserPasswordValid,
105 isUserPasswordValidOrEmpty,
106 isUserVideoLanguages,
107 isUserBlockedReasonValid,
108 isUserRoleValid,
109 isUserVideoQuotaValid,
110 isUserVideoQuotaDailyValid,
111 isUserUsernameValid,
112 isUserAdminFlagsValid,
113 isUserEmailVerifiedValid,
114 isUserNSFWPolicyValid,
115 isUserP2PEnabledValid,
116 isUserAutoPlayVideoValid,
117 isUserAutoPlayNextVideoValid,
118 isUserAutoPlayNextVideoPlaylistValid,
119 isUserDisplayNameValid,
120 isUserDescriptionValid,
121 isUserEmailPublicValid,
122 isUserNoModal
123}
diff --git a/server/helpers/custom-validators/video-blacklist.ts b/server/helpers/custom-validators/video-blacklist.ts
deleted file mode 100644
index 34fcec38e..000000000
--- a/server/helpers/custom-validators/video-blacklist.ts
+++ /dev/null
@@ -1,22 +0,0 @@
1import validator from 'validator'
2import { exists } from './misc'
3import { CONSTRAINTS_FIELDS } from '../../initializers/constants'
4import { VideoBlacklistType } from '../../../shared/models/videos'
5
6const VIDEO_BLACKLIST_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_BLACKLIST
7
8function isVideoBlacklistReasonValid (value: string) {
9 return value === null || validator.isLength(value, VIDEO_BLACKLIST_CONSTRAINTS_FIELDS.REASON)
10}
11
12function isVideoBlacklistTypeValid (value: any) {
13 return exists(value) &&
14 (value === VideoBlacklistType.AUTO_BEFORE_PUBLISHED || value === VideoBlacklistType.MANUAL)
15}
16
17// ---------------------------------------------------------------------------
18
19export {
20 isVideoBlacklistReasonValid,
21 isVideoBlacklistTypeValid
22}
diff --git a/server/helpers/custom-validators/video-captions.ts b/server/helpers/custom-validators/video-captions.ts
deleted file mode 100644
index 0e24655a0..000000000
--- a/server/helpers/custom-validators/video-captions.ts
+++ /dev/null
@@ -1,43 +0,0 @@
1import { UploadFilesForCheck } from 'express'
2import { readFile } from 'fs-extra'
3import { getFileSize } from '@shared/extra-utils'
4import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_LANGUAGES } from '../../initializers/constants'
5import { logger } from '../logger'
6import { exists, isFileValid } from './misc'
7
8function isVideoCaptionLanguageValid (value: any) {
9 return exists(value) && VIDEO_LANGUAGES[value] !== undefined
10}
11
12// MacOS sends application/octet-stream
13const videoCaptionTypesRegex = [ ...Object.keys(MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT), 'application/octet-stream' ]
14 .map(m => `(${m})`)
15 .join('|')
16
17function isVideoCaptionFile (files: UploadFilesForCheck, field: string) {
18 return isFileValid({
19 files,
20 mimeTypeRegex: videoCaptionTypesRegex,
21 field,
22 maxSize: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max
23 })
24}
25
26async function isVTTFileValid (filePath: string) {
27 const size = await getFileSize(filePath)
28 const content = await readFile(filePath, 'utf8')
29
30 logger.debug('Checking VTT file %s', filePath, { size, content })
31
32 if (size > CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max) return false
33
34 return content?.startsWith('WEBVTT')
35}
36
37// ---------------------------------------------------------------------------
38
39export {
40 isVideoCaptionFile,
41 isVTTFileValid,
42 isVideoCaptionLanguageValid
43}
diff --git a/server/helpers/custom-validators/video-channel-syncs.ts b/server/helpers/custom-validators/video-channel-syncs.ts
deleted file mode 100644
index c5a9afa96..000000000
--- a/server/helpers/custom-validators/video-channel-syncs.ts
+++ /dev/null
@@ -1,6 +0,0 @@
1import { VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants'
2import { exists } from './misc'
3
4export function isVideoChannelSyncStateValid (value: any) {
5 return exists(value) && VIDEO_CHANNEL_SYNC_STATE[value] !== undefined
6}
diff --git a/server/helpers/custom-validators/video-channels.ts b/server/helpers/custom-validators/video-channels.ts
deleted file mode 100644
index 249083f39..000000000
--- a/server/helpers/custom-validators/video-channels.ts
+++ /dev/null
@@ -1,32 +0,0 @@
1import validator from 'validator'
2import { CONSTRAINTS_FIELDS } from '../../initializers/constants'
3import { exists } from './misc'
4import { isUserUsernameValid } from './users'
5
6const VIDEO_CHANNELS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_CHANNELS
7
8function isVideoChannelUsernameValid (value: string) {
9 // Use the same constraints than user username
10 return isUserUsernameValid(value)
11}
12
13function isVideoChannelDescriptionValid (value: string) {
14 return value === null || validator.isLength(value, VIDEO_CHANNELS_CONSTRAINTS_FIELDS.DESCRIPTION)
15}
16
17function isVideoChannelDisplayNameValid (value: string) {
18 return exists(value) && validator.isLength(value, VIDEO_CHANNELS_CONSTRAINTS_FIELDS.NAME)
19}
20
21function isVideoChannelSupportValid (value: string) {
22 return value === null || (exists(value) && validator.isLength(value, VIDEO_CHANNELS_CONSTRAINTS_FIELDS.SUPPORT))
23}
24
25// ---------------------------------------------------------------------------
26
27export {
28 isVideoChannelUsernameValid,
29 isVideoChannelDescriptionValid,
30 isVideoChannelDisplayNameValid,
31 isVideoChannelSupportValid
32}
diff --git a/server/helpers/custom-validators/video-comments.ts b/server/helpers/custom-validators/video-comments.ts
deleted file mode 100644
index 94bdf237a..000000000
--- a/server/helpers/custom-validators/video-comments.ts
+++ /dev/null
@@ -1,14 +0,0 @@
1import validator from 'validator'
2import { CONSTRAINTS_FIELDS } from '../../initializers/constants'
3
4const VIDEO_COMMENTS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_COMMENTS
5
6function isValidVideoCommentText (value: string) {
7 return value === null || validator.isLength(value, VIDEO_COMMENTS_CONSTRAINTS_FIELDS.TEXT)
8}
9
10// ---------------------------------------------------------------------------
11
12export {
13 isValidVideoCommentText
14}
diff --git a/server/helpers/custom-validators/video-imports.ts b/server/helpers/custom-validators/video-imports.ts
deleted file mode 100644
index da8962cb6..000000000
--- a/server/helpers/custom-validators/video-imports.ts
+++ /dev/null
@@ -1,46 +0,0 @@
1import 'multer'
2import { UploadFilesForCheck } from 'express'
3import validator from 'validator'
4import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_IMPORT_STATES } from '../../initializers/constants'
5import { exists, isFileValid } from './misc'
6
7function isVideoImportTargetUrlValid (url: string) {
8 const isURLOptions = {
9 require_host: true,
10 require_tld: true,
11 require_protocol: true,
12 require_valid_protocol: true,
13 protocols: [ 'http', 'https' ]
14 }
15
16 return exists(url) &&
17 validator.isURL('' + url, isURLOptions) &&
18 validator.isLength('' + url, CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL)
19}
20
21function isVideoImportStateValid (value: any) {
22 return exists(value) && VIDEO_IMPORT_STATES[value] !== undefined
23}
24
25// MacOS sends application/octet-stream
26const videoTorrentImportRegex = [ ...Object.keys(MIMETYPES.TORRENT.MIMETYPE_EXT), 'application/octet-stream' ]
27 .map(m => `(${m})`)
28 .join('|')
29
30function isVideoImportTorrentFile (files: UploadFilesForCheck) {
31 return isFileValid({
32 files,
33 mimeTypeRegex: videoTorrentImportRegex,
34 field: 'torrentfile',
35 maxSize: CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_FILE.FILE_SIZE.max,
36 optional: true
37 })
38}
39
40// ---------------------------------------------------------------------------
41
42export {
43 isVideoImportStateValid,
44 isVideoImportTargetUrlValid,
45 isVideoImportTorrentFile
46}
diff --git a/server/helpers/custom-validators/video-lives.ts b/server/helpers/custom-validators/video-lives.ts
deleted file mode 100644
index 69d08ae68..000000000
--- a/server/helpers/custom-validators/video-lives.ts
+++ /dev/null
@@ -1,11 +0,0 @@
1import { LiveVideoLatencyMode } from '@shared/models'
2
3function isLiveLatencyModeValid (value: any) {
4 return [ LiveVideoLatencyMode.DEFAULT, LiveVideoLatencyMode.SMALL_LATENCY, LiveVideoLatencyMode.HIGH_LATENCY ].includes(value)
5}
6
7// ---------------------------------------------------------------------------
8
9export {
10 isLiveLatencyModeValid
11}
diff --git a/server/helpers/custom-validators/video-ownership.ts b/server/helpers/custom-validators/video-ownership.ts
deleted file mode 100644
index cf15b385a..000000000
--- a/server/helpers/custom-validators/video-ownership.ts
+++ /dev/null
@@ -1,20 +0,0 @@
1import { Response } from 'express'
2import { MUserId } from '@server/types/models'
3import { MVideoChangeOwnershipFull } from '@server/types/models/video/video-change-ownership'
4import { HttpStatusCode } from '../../../shared/models/http/http-error-codes'
5
6function checkUserCanTerminateOwnershipChange (user: MUserId, videoChangeOwnership: MVideoChangeOwnershipFull, res: Response) {
7 if (videoChangeOwnership.NextOwner.userId === user.id) {
8 return true
9 }
10
11 res.fail({
12 status: HttpStatusCode.FORBIDDEN_403,
13 message: 'Cannot terminate an ownership change of another user'
14 })
15 return false
16}
17
18export {
19 checkUserCanTerminateOwnershipChange
20}
diff --git a/server/helpers/custom-validators/video-playlists.ts b/server/helpers/custom-validators/video-playlists.ts
deleted file mode 100644
index 180018fc5..000000000
--- a/server/helpers/custom-validators/video-playlists.ts
+++ /dev/null
@@ -1,35 +0,0 @@
1import { exists } from './misc'
2import validator from 'validator'
3import { CONSTRAINTS_FIELDS, VIDEO_PLAYLIST_PRIVACIES, VIDEO_PLAYLIST_TYPES } from '../../initializers/constants'
4
5const PLAYLISTS_CONSTRAINT_FIELDS = CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS
6
7function isVideoPlaylistNameValid (value: any) {
8 return exists(value) && validator.isLength(value, PLAYLISTS_CONSTRAINT_FIELDS.NAME)
9}
10
11function isVideoPlaylistDescriptionValid (value: any) {
12 return value === null || (exists(value) && validator.isLength(value, PLAYLISTS_CONSTRAINT_FIELDS.DESCRIPTION))
13}
14
15function isVideoPlaylistPrivacyValid (value: number) {
16 return validator.isInt(value + '') && VIDEO_PLAYLIST_PRIVACIES[value] !== undefined
17}
18
19function isVideoPlaylistTimestampValid (value: any) {
20 return value === null || (exists(value) && validator.isInt('' + value, { min: 0 }))
21}
22
23function isVideoPlaylistTypeValid (value: any) {
24 return exists(value) && VIDEO_PLAYLIST_TYPES[value] !== undefined
25}
26
27// ---------------------------------------------------------------------------
28
29export {
30 isVideoPlaylistNameValid,
31 isVideoPlaylistDescriptionValid,
32 isVideoPlaylistPrivacyValid,
33 isVideoPlaylistTimestampValid,
34 isVideoPlaylistTypeValid
35}
diff --git a/server/helpers/custom-validators/video-rates.ts b/server/helpers/custom-validators/video-rates.ts
deleted file mode 100644
index f2b6f7cae..000000000
--- a/server/helpers/custom-validators/video-rates.ts
+++ /dev/null
@@ -1,5 +0,0 @@
1function isRatingValid (value: any) {
2 return value === 'like' || value === 'dislike'
3}
4
5export { isRatingValid }
diff --git a/server/helpers/custom-validators/video-redundancies.ts b/server/helpers/custom-validators/video-redundancies.ts
deleted file mode 100644
index 50a559c4f..000000000
--- a/server/helpers/custom-validators/video-redundancies.ts
+++ /dev/null
@@ -1,12 +0,0 @@
1import { exists } from './misc'
2
3function isVideoRedundancyTarget (value: any) {
4 return exists(value) &&
5 (value === 'my-videos' || value === 'remote-videos')
6}
7
8// ---------------------------------------------------------------------------
9
10export {
11 isVideoRedundancyTarget
12}
diff --git a/server/helpers/custom-validators/video-stats.ts b/server/helpers/custom-validators/video-stats.ts
deleted file mode 100644
index 1e22f0654..000000000
--- a/server/helpers/custom-validators/video-stats.ts
+++ /dev/null
@@ -1,16 +0,0 @@
1import { VideoStatsTimeserieMetric } from '@shared/models'
2
3const validMetrics = new Set<VideoStatsTimeserieMetric>([
4 'viewers',
5 'aggregateWatchTime'
6])
7
8function isValidStatTimeserieMetric (value: VideoStatsTimeserieMetric) {
9 return validMetrics.has(value)
10}
11
12// ---------------------------------------------------------------------------
13
14export {
15 isValidStatTimeserieMetric
16}
diff --git a/server/helpers/custom-validators/video-studio.ts b/server/helpers/custom-validators/video-studio.ts
deleted file mode 100644
index 68dfec8dd..000000000
--- a/server/helpers/custom-validators/video-studio.ts
+++ /dev/null
@@ -1,53 +0,0 @@
1import validator from 'validator'
2import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
3import { buildTaskFileFieldname } from '@server/lib/video-studio'
4import { VideoStudioTask } from '@shared/models'
5import { isArray } from './misc'
6import { isVideoFileMimeTypeValid, isVideoImageValid } from './videos'
7import { forceNumber } from '@shared/core-utils'
8
9function isValidStudioTasksArray (tasks: any) {
10 if (!isArray(tasks)) return false
11
12 return tasks.length >= CONSTRAINTS_FIELDS.VIDEO_STUDIO.TASKS.min &&
13 tasks.length <= CONSTRAINTS_FIELDS.VIDEO_STUDIO.TASKS.max
14}
15
16function isStudioCutTaskValid (task: VideoStudioTask) {
17 if (task.name !== 'cut') return false
18 if (!task.options) return false
19
20 const { start, end } = task.options
21 if (!start && !end) return false
22
23 if (start && !validator.isInt(start + '', CONSTRAINTS_FIELDS.VIDEO_STUDIO.CUT_TIME)) return false
24 if (end && !validator.isInt(end + '', CONSTRAINTS_FIELDS.VIDEO_STUDIO.CUT_TIME)) return false
25
26 if (!start || !end) return true
27
28 return forceNumber(start) < forceNumber(end)
29}
30
31function isStudioTaskAddIntroOutroValid (task: VideoStudioTask, indice: number, files: Express.Multer.File[]) {
32 const file = files.find(f => f.fieldname === buildTaskFileFieldname(indice, 'file'))
33
34 return (task.name === 'add-intro' || task.name === 'add-outro') &&
35 file && isVideoFileMimeTypeValid([ file ], null)
36}
37
38function isStudioTaskAddWatermarkValid (task: VideoStudioTask, indice: number, files: Express.Multer.File[]) {
39 const file = files.find(f => f.fieldname === buildTaskFileFieldname(indice, 'file'))
40
41 return task.name === 'add-watermark' &&
42 file && isVideoImageValid([ file ], null, true)
43}
44
45// ---------------------------------------------------------------------------
46
47export {
48 isValidStudioTasksArray,
49
50 isStudioCutTaskValid,
51 isStudioTaskAddIntroOutroValid,
52 isStudioTaskAddWatermarkValid
53}
diff --git a/server/helpers/custom-validators/video-transcoding.ts b/server/helpers/custom-validators/video-transcoding.ts
deleted file mode 100644
index 220530de4..000000000
--- a/server/helpers/custom-validators/video-transcoding.ts
+++ /dev/null
@@ -1,12 +0,0 @@
1import { exists } from './misc'
2
3function isValidCreateTranscodingType (value: any) {
4 return exists(value) &&
5 (value === 'hls' || value === 'webtorrent' || value === 'web-video') // TODO: remove webtorrent in v7
6}
7
8// ---------------------------------------------------------------------------
9
10export {
11 isValidCreateTranscodingType
12}
diff --git a/server/helpers/custom-validators/video-view.ts b/server/helpers/custom-validators/video-view.ts
deleted file mode 100644
index 091c92083..000000000
--- a/server/helpers/custom-validators/video-view.ts
+++ /dev/null
@@ -1,12 +0,0 @@
1import { exists } from './misc'
2
3function isVideoTimeValid (value: number, videoDuration?: number) {
4 if (value < 0) return false
5 if (exists(videoDuration) && value > videoDuration) return false
6
7 return true
8}
9
10export {
11 isVideoTimeValid
12}
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts
deleted file mode 100644
index 00c6deed4..000000000
--- a/server/helpers/custom-validators/videos.ts
+++ /dev/null
@@ -1,218 +0,0 @@
1import { Request, Response, UploadFilesForCheck } from 'express'
2import { decode as magnetUriDecode } from 'magnet-uri'
3import validator from 'validator'
4import { getVideoWithAttributes } from '@server/helpers/video'
5import { HttpStatusCode, VideoInclude, VideoPrivacy, VideoRateType } from '@shared/models'
6import {
7 CONSTRAINTS_FIELDS,
8 MIMETYPES,
9 VIDEO_CATEGORIES,
10 VIDEO_LICENCES,
11 VIDEO_LIVE,
12 VIDEO_PRIVACIES,
13 VIDEO_RATE_TYPES,
14 VIDEO_STATES
15} from '../../initializers/constants'
16import { exists, isArray, isDateValid, isFileValid } from './misc'
17
18const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS
19
20function isVideoIncludeValid (include: VideoInclude) {
21 return exists(include) && validator.isInt('' + include)
22}
23
24function isVideoCategoryValid (value: any) {
25 return value === null || VIDEO_CATEGORIES[value] !== undefined
26}
27
28function isVideoStateValid (value: any) {
29 return exists(value) && VIDEO_STATES[value] !== undefined
30}
31
32function isVideoLicenceValid (value: any) {
33 return value === null || VIDEO_LICENCES[value] !== undefined
34}
35
36function isVideoLanguageValid (value: any) {
37 return value === null ||
38 (typeof value === 'string' && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.LANGUAGE))
39}
40
41function isVideoDurationValid (value: string) {
42 return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION)
43}
44
45function isVideoDescriptionValid (value: string) {
46 return value === null || (exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.DESCRIPTION))
47}
48
49function isVideoSupportValid (value: string) {
50 return value === null || (exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.SUPPORT))
51}
52
53function isVideoNameValid (value: string) {
54 return exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.NAME)
55}
56
57function isVideoTagValid (tag: string) {
58 return exists(tag) && validator.isLength(tag, VIDEOS_CONSTRAINTS_FIELDS.TAG)
59}
60
61function areVideoTagsValid (tags: string[]) {
62 return tags === null || (
63 isArray(tags) &&
64 validator.isInt(tags.length.toString(), VIDEOS_CONSTRAINTS_FIELDS.TAGS) &&
65 tags.every(tag => isVideoTagValid(tag))
66 )
67}
68
69function isVideoViewsValid (value: string) {
70 return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.VIEWS)
71}
72
73const ratingTypes = new Set(Object.values(VIDEO_RATE_TYPES))
74function isVideoRatingTypeValid (value: string) {
75 return value === 'none' || ratingTypes.has(value as VideoRateType)
76}
77
78function isVideoFileExtnameValid (value: string) {
79 return exists(value) && (value === VIDEO_LIVE.EXTENSION || MIMETYPES.VIDEO.EXT_MIMETYPE[value] !== undefined)
80}
81
82function isVideoFileMimeTypeValid (files: UploadFilesForCheck, field = 'videofile') {
83 return isFileValid({
84 files,
85 mimeTypeRegex: MIMETYPES.VIDEO.MIMETYPES_REGEX,
86 field,
87 maxSize: null
88 })
89}
90
91const videoImageTypes = CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME
92 .map(v => v.replace('.', ''))
93 .join('|')
94const videoImageTypesRegex = `image/(${videoImageTypes})`
95
96function isVideoImageValid (files: UploadFilesForCheck, field: string, optional = true) {
97 return isFileValid({
98 files,
99 mimeTypeRegex: videoImageTypesRegex,
100 field,
101 maxSize: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max,
102 optional
103 })
104}
105
106function isVideoPrivacyValid (value: number) {
107 return VIDEO_PRIVACIES[value] !== undefined
108}
109
110function isVideoReplayPrivacyValid (value: number) {
111 return VIDEO_PRIVACIES[value] !== undefined && value !== VideoPrivacy.PASSWORD_PROTECTED
112}
113
114function isScheduleVideoUpdatePrivacyValid (value: number) {
115 return value === VideoPrivacy.UNLISTED || value === VideoPrivacy.PUBLIC || value === VideoPrivacy.INTERNAL
116}
117
118function isVideoOriginallyPublishedAtValid (value: string | null) {
119 return value === null || isDateValid(value)
120}
121
122function isVideoFileInfoHashValid (value: string | null | undefined) {
123 return exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.INFO_HASH)
124}
125
126function isVideoFileResolutionValid (value: string) {
127 return exists(value) && validator.isInt(value + '')
128}
129
130function isVideoFPSResolutionValid (value: string) {
131 return value === null || validator.isInt(value + '')
132}
133
134function isVideoFileSizeValid (value: string) {
135 return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.FILE_SIZE)
136}
137
138function isVideoMagnetUriValid (value: string) {
139 if (!exists(value)) return false
140
141 const parsed = magnetUriDecode(value)
142 return parsed && isVideoFileInfoHashValid(parsed.infoHash)
143}
144
145function isPasswordValid (password: string) {
146 return password.length >= CONSTRAINTS_FIELDS.VIDEO_PASSWORD.LENGTH.min &&
147 password.length < CONSTRAINTS_FIELDS.VIDEO_PASSWORD.LENGTH.max
148}
149
150function isValidPasswordProtectedPrivacy (req: Request, res: Response) {
151 const fail = (message: string) => {
152 res.fail({
153 status: HttpStatusCode.BAD_REQUEST_400,
154 message
155 })
156 return false
157 }
158
159 let privacy: VideoPrivacy
160 const video = getVideoWithAttributes(res)
161
162 if (exists(req.body?.privacy)) privacy = req.body.privacy
163 else if (exists(video?.privacy)) privacy = video.privacy
164
165 if (privacy !== VideoPrivacy.PASSWORD_PROTECTED) return true
166
167 if (!exists(req.body.videoPasswords) && !exists(req.body.passwords)) return fail('Video passwords are missing.')
168
169 const passwords = req.body.videoPasswords || req.body.passwords
170
171 if (passwords.length === 0) return fail('At least one video password is required.')
172
173 if (new Set(passwords).size !== passwords.length) return fail('Duplicate video passwords are not allowed.')
174
175 for (const password of passwords) {
176 if (typeof password !== 'string') {
177 return fail('Video password should be a string.')
178 }
179
180 if (!isPasswordValid(password)) {
181 return fail('Invalid video password. Password length should be at least 2 characters and no more than 100 characters.')
182 }
183 }
184
185 return true
186}
187
188// ---------------------------------------------------------------------------
189
190export {
191 isVideoCategoryValid,
192 isVideoLicenceValid,
193 isVideoLanguageValid,
194 isVideoDescriptionValid,
195 isVideoFileInfoHashValid,
196 isVideoNameValid,
197 areVideoTagsValid,
198 isVideoFPSResolutionValid,
199 isScheduleVideoUpdatePrivacyValid,
200 isVideoOriginallyPublishedAtValid,
201 isVideoMagnetUriValid,
202 isVideoStateValid,
203 isVideoIncludeValid,
204 isVideoViewsValid,
205 isVideoRatingTypeValid,
206 isVideoFileExtnameValid,
207 isVideoFileMimeTypeValid,
208 isVideoDurationValid,
209 isVideoTagValid,
210 isVideoPrivacyValid,
211 isVideoReplayPrivacyValid,
212 isVideoFileResolutionValid,
213 isVideoFileSizeValid,
214 isVideoImageValid,
215 isVideoSupportValid,
216 isPasswordValid,
217 isValidPasswordProtectedPrivacy
218}
diff --git a/server/helpers/custom-validators/webfinger.ts b/server/helpers/custom-validators/webfinger.ts
deleted file mode 100644
index dd914341e..000000000
--- a/server/helpers/custom-validators/webfinger.ts
+++ /dev/null
@@ -1,21 +0,0 @@
1import { REMOTE_SCHEME, WEBSERVER } from '../../initializers/constants'
2import { sanitizeHost } from '../core-utils'
3import { exists } from './misc'
4
5function isWebfingerLocalResourceValid (value: string) {
6 if (!exists(value)) return false
7 if (value.startsWith('acct:') === false) return false
8
9 const actorWithHost = value.substr(5)
10 const actorParts = actorWithHost.split('@')
11 if (actorParts.length !== 2) return false
12
13 const host = actorParts[1]
14 return sanitizeHost(host, REMOTE_SCHEME.HTTP) === WEBSERVER.HOST
15}
16
17// ---------------------------------------------------------------------------
18
19export {
20 isWebfingerLocalResourceValid
21}
diff --git a/server/helpers/database-utils.ts b/server/helpers/database-utils.ts
deleted file mode 100644
index b6ba7fd75..000000000
--- a/server/helpers/database-utils.ts
+++ /dev/null
@@ -1,121 +0,0 @@
1import retry from 'async/retry'
2import Bluebird from 'bluebird'
3import { Transaction } from 'sequelize'
4import { Model } from 'sequelize-typescript'
5import { sequelizeTypescript } from '@server/initializers/database'
6import { logger } from './logger'
7
8function retryTransactionWrapper <T, A, B, C, D> (
9 functionToRetry: (arg1: A, arg2: B, arg3: C, arg4: D) => Promise<T>,
10 arg1: A,
11 arg2: B,
12 arg3: C,
13 arg4: D,
14): Promise<T>
15
16function retryTransactionWrapper <T, A, B, C> (
17 functionToRetry: (arg1: A, arg2: B, arg3: C) => Promise<T>,
18 arg1: A,
19 arg2: B,
20 arg3: C
21): Promise<T>
22
23function retryTransactionWrapper <T, A, B> (
24 functionToRetry: (arg1: A, arg2: B) => Promise<T>,
25 arg1: A,
26 arg2: B
27): Promise<T>
28
29function retryTransactionWrapper <T, A> (
30 functionToRetry: (arg1: A) => Promise<T>,
31 arg1: A
32): Promise<T>
33
34function retryTransactionWrapper <T> (
35 functionToRetry: () => Promise<T> | Bluebird<T>
36): Promise<T>
37
38function retryTransactionWrapper <T> (
39 functionToRetry: (...args: any[]) => Promise<T>,
40 ...args: any[]
41): Promise<T> {
42 return transactionRetryer<T>(callback => {
43 functionToRetry.apply(null, args)
44 .then((result: T) => callback(null, result))
45 .catch(err => callback(err))
46 })
47 .catch(err => {
48 logger.warn(`Cannot execute ${functionToRetry.name} with many retries.`, { err })
49 throw err
50 })
51}
52
53function transactionRetryer <T> (func: (err: any, data: T) => any) {
54 return new Promise<T>((res, rej) => {
55 retry(
56 {
57 times: 5,
58
59 errorFilter: err => {
60 const willRetry = (err.name === 'SequelizeDatabaseError')
61 logger.debug('Maybe retrying the transaction function.', { willRetry, err, tags: [ 'sql', 'retry' ] })
62 return willRetry
63 }
64 },
65 func,
66 (err, data) => err ? rej(err) : res(data)
67 )
68 })
69}
70
71function saveInTransactionWithRetries <T extends Pick<Model, 'save'>> (model: T) {
72 return retryTransactionWrapper(() => {
73 return sequelizeTypescript.transaction(async transaction => {
74 await model.save({ transaction })
75 })
76 })
77}
78
79// ---------------------------------------------------------------------------
80
81function resetSequelizeInstance <T> (instance: Model<T>) {
82 return instance.reload()
83}
84
85function filterNonExistingModels <T extends { hasSameUniqueKeysThan (other: T): boolean }> (
86 fromDatabase: T[],
87 newModels: T[]
88) {
89 return fromDatabase.filter(f => !newModels.find(newModel => newModel.hasSameUniqueKeysThan(f)))
90}
91
92function deleteAllModels <T extends Pick<Model, 'destroy'>> (models: T[], transaction: Transaction) {
93 return Promise.all(models.map(f => f.destroy({ transaction })))
94}
95
96// ---------------------------------------------------------------------------
97
98function runInReadCommittedTransaction <T> (fn: (t: Transaction) => Promise<T>) {
99 const options = { isolationLevel: Transaction.ISOLATION_LEVELS.READ_COMMITTED }
100
101 return sequelizeTypescript.transaction(options, t => fn(t))
102}
103
104function afterCommitIfTransaction (t: Transaction, fn: Function) {
105 if (t) return t.afterCommit(() => fn())
106
107 return fn()
108}
109
110// ---------------------------------------------------------------------------
111
112export {
113 resetSequelizeInstance,
114 retryTransactionWrapper,
115 transactionRetryer,
116 saveInTransactionWithRetries,
117 afterCommitIfTransaction,
118 filterNonExistingModels,
119 deleteAllModels,
120 runInReadCommittedTransaction
121}
diff --git a/server/helpers/debounce.ts b/server/helpers/debounce.ts
deleted file mode 100644
index 77d99a894..000000000
--- a/server/helpers/debounce.ts
+++ /dev/null
@@ -1,16 +0,0 @@
1export function Debounce (config: { timeoutMS: number }) {
2 let timeoutRef: NodeJS.Timeout
3
4 return function (_target, _key, descriptor: PropertyDescriptor) {
5 const original = descriptor.value
6
7 descriptor.value = function (...args: any[]) {
8 clearTimeout(timeoutRef)
9
10 timeoutRef = setTimeout(() => {
11 original.apply(this, args)
12
13 }, config.timeoutMS)
14 }
15 }
16}
diff --git a/server/helpers/decache.ts b/server/helpers/decache.ts
deleted file mode 100644
index 6be446ff6..000000000
--- a/server/helpers/decache.ts
+++ /dev/null
@@ -1,78 +0,0 @@
1// Thanks: https://github.com/dwyl/decache
2// We reuse this file to also uncache plugin base path
3
4import { extname } from 'path'
5
6function decachePlugin (libraryPath: string) {
7 const moduleName = find(libraryPath)
8
9 if (!moduleName) return
10
11 searchCache(moduleName, function (mod) {
12 delete require.cache[mod.id]
13
14 removeCachedPath(mod.path)
15 })
16}
17
18function decacheModule (name: string) {
19 const moduleName = find(name)
20
21 if (!moduleName) return
22
23 searchCache(moduleName, function (mod) {
24 delete require.cache[mod.id]
25
26 removeCachedPath(mod.path)
27 })
28}
29
30// ---------------------------------------------------------------------------
31
32export {
33 decacheModule,
34 decachePlugin
35}
36
37// ---------------------------------------------------------------------------
38
39function find (moduleName: string) {
40 try {
41 return require.resolve(moduleName)
42 } catch {
43 return ''
44 }
45}
46
47function searchCache (moduleName: string, callback: (current: NodeModule) => void) {
48 const resolvedModule = require.resolve(moduleName)
49 let mod: NodeModule
50 const visited = {}
51
52 if (resolvedModule && ((mod = require.cache[resolvedModule]) !== undefined)) {
53 // Recursively go over the results
54 (function run (current) {
55 visited[current.id] = true
56
57 current.children.forEach(function (child) {
58 if (extname(child.filename) !== '.node' && !visited[child.id]) {
59 run(child)
60 }
61 })
62
63 // Call the specified callback providing the
64 // found module
65 callback(current)
66 })(mod)
67 }
68};
69
70function removeCachedPath (pluginPath: string) {
71 const pathCache = (module.constructor as any)._pathCache as { [ id: string ]: string[] }
72
73 Object.keys(pathCache).forEach(function (cacheKey) {
74 if (cacheKey.includes(pluginPath)) {
75 delete pathCache[cacheKey]
76 }
77 })
78}
diff --git a/server/helpers/dns.ts b/server/helpers/dns.ts
deleted file mode 100644
index da8b666c2..000000000
--- a/server/helpers/dns.ts
+++ /dev/null
@@ -1,29 +0,0 @@
1import { lookup } from 'dns'
2import { parse as parseIP } from 'ipaddr.js'
3
4function dnsLookupAll (hostname: string) {
5 return new Promise<string[]>((res, rej) => {
6 lookup(hostname, { family: 0, all: true }, (err, adresses) => {
7 if (err) return rej(err)
8
9 return res(adresses.map(a => a.address))
10 })
11 })
12}
13
14async function isResolvingToUnicastOnly (hostname: string) {
15 const addresses = await dnsLookupAll(hostname)
16
17 for (const address of addresses) {
18 const parsed = parseIP(address)
19
20 if (parsed.range() !== 'unicast') return false
21 }
22
23 return true
24}
25
26export {
27 dnsLookupAll,
28 isResolvingToUnicastOnly
29}
diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts
deleted file mode 100644
index 783097e55..000000000
--- a/server/helpers/express-utils.ts
+++ /dev/null
@@ -1,156 +0,0 @@
1import express, { RequestHandler } from 'express'
2import multer, { diskStorage } from 'multer'
3import { getLowercaseExtension } from '@shared/core-utils'
4import { CONFIG } from '../initializers/config'
5import { REMOTE_SCHEME } from '../initializers/constants'
6import { isArray } from './custom-validators/misc'
7import { logger } from './logger'
8import { deleteFileAndCatch, generateRandomString } from './utils'
9import { getExtFromMimetype } from './video'
10
11function buildNSFWFilter (res?: express.Response, paramNSFW?: string) {
12 if (paramNSFW === 'true') return true
13 if (paramNSFW === 'false') return false
14 if (paramNSFW === 'both') return undefined
15
16 if (res?.locals.oauth) {
17 const user = res.locals.oauth.token.User
18
19 // User does not want NSFW videos
20 if (user.nsfwPolicy === 'do_not_list') return false
21
22 // Both
23 return undefined
24 }
25
26 if (CONFIG.INSTANCE.DEFAULT_NSFW_POLICY === 'do_not_list') return false
27
28 // Display all
29 return null
30}
31
32function cleanUpReqFiles (req: express.Request) {
33 const filesObject = req.files
34 if (!filesObject) return
35
36 if (isArray(filesObject)) {
37 filesObject.forEach(f => deleteFileAndCatch(f.path))
38 return
39 }
40
41 for (const key of Object.keys(filesObject)) {
42 const files = filesObject[key]
43
44 files.forEach(f => deleteFileAndCatch(f.path))
45 }
46}
47
48function getHostWithPort (host: string) {
49 const splitted = host.split(':')
50
51 // The port was not specified
52 if (splitted.length === 1) {
53 if (REMOTE_SCHEME.HTTP === 'https') return host + ':443'
54
55 return host + ':80'
56 }
57
58 return host
59}
60
61function createReqFiles (
62 fieldNames: string[],
63 mimeTypes: { [id: string]: string | string[] },
64 destination = CONFIG.STORAGE.TMP_DIR
65): RequestHandler {
66 const storage = diskStorage({
67 destination: (req, file, cb) => {
68 cb(null, destination)
69 },
70
71 filename: (req, file, cb) => {
72 return generateReqFilename(file, mimeTypes, cb)
73 }
74 })
75
76 const fields: { name: string, maxCount: number }[] = []
77 for (const fieldName of fieldNames) {
78 fields.push({
79 name: fieldName,
80 maxCount: 1
81 })
82 }
83
84 return multer({ storage }).fields(fields)
85}
86
87function createAnyReqFiles (
88 mimeTypes: { [id: string]: string | string[] },
89 fileFilter: (req: express.Request, file: Express.Multer.File, cb: (err: Error, result: boolean) => void) => void
90): RequestHandler {
91 const storage = diskStorage({
92 destination: (req, file, cb) => {
93 cb(null, CONFIG.STORAGE.TMP_DIR)
94 },
95
96 filename: (req, file, cb) => {
97 return generateReqFilename(file, mimeTypes, cb)
98 }
99 })
100
101 return multer({ storage, fileFilter }).any()
102}
103
104function isUserAbleToSearchRemoteURI (res: express.Response) {
105 const user = res.locals.oauth ? res.locals.oauth.token.User : undefined
106
107 return CONFIG.SEARCH.REMOTE_URI.ANONYMOUS === true ||
108 (CONFIG.SEARCH.REMOTE_URI.USERS === true && user !== undefined)
109}
110
111function getCountVideos (req: express.Request) {
112 return req.query.skipCount !== true
113}
114
115// ---------------------------------------------------------------------------
116
117export {
118 buildNSFWFilter,
119 getHostWithPort,
120 createAnyReqFiles,
121 isUserAbleToSearchRemoteURI,
122 createReqFiles,
123 cleanUpReqFiles,
124 getCountVideos
125}
126
127// ---------------------------------------------------------------------------
128
129async function generateReqFilename (
130 file: Express.Multer.File,
131 mimeTypes: { [id: string]: string | string[] },
132 cb: (err: Error, name: string) => void
133) {
134 let extension: string
135 const fileExtension = getLowercaseExtension(file.originalname)
136 const extensionFromMimetype = getExtFromMimetype(mimeTypes, file.mimetype)
137
138 // Take the file extension if we don't understand the mime type
139 if (!extensionFromMimetype) {
140 extension = fileExtension
141 } else {
142 // Take the first available extension for this mimetype
143 extension = extensionFromMimetype
144 }
145
146 let randomString = ''
147
148 try {
149 randomString = await generateRandomString(16)
150 } catch (err) {
151 logger.error('Cannot generate random string for file name.', { err })
152 randomString = 'fake-random-string'
153 }
154
155 cb(null, randomString + extension)
156}
diff --git a/server/helpers/ffmpeg/codecs.ts b/server/helpers/ffmpeg/codecs.ts
deleted file mode 100644
index 3bd7db396..000000000
--- a/server/helpers/ffmpeg/codecs.ts
+++ /dev/null
@@ -1,64 +0,0 @@
1import { FfprobeData } from 'fluent-ffmpeg'
2import { getAudioStream, getVideoStream } from '@shared/ffmpeg'
3import { logger } from '../logger'
4import { forceNumber } from '@shared/core-utils'
5
6export async function getVideoStreamCodec (path: string) {
7 const videoStream = await getVideoStream(path)
8 if (!videoStream) return ''
9
10 const videoCodec = videoStream.codec_tag_string
11
12 if (videoCodec === 'vp09') return 'vp09.00.50.08'
13 if (videoCodec === 'hev1') return 'hev1.1.6.L93.B0'
14
15 const baseProfileMatrix = {
16 avc1: {
17 High: '6400',
18 Main: '4D40',
19 Baseline: '42E0'
20 },
21 av01: {
22 High: '1',
23 Main: '0',
24 Professional: '2'
25 }
26 }
27
28 let baseProfile = baseProfileMatrix[videoCodec][videoStream.profile]
29 if (!baseProfile) {
30 logger.warn('Cannot get video profile codec of %s.', path, { videoStream })
31 baseProfile = baseProfileMatrix[videoCodec]['High'] // Fallback
32 }
33
34 if (videoCodec === 'av01') {
35 let level = videoStream.level.toString()
36 if (level.length === 1) level = `0${level}`
37
38 // Guess the tier indicator and bit depth
39 return `${videoCodec}.${baseProfile}.${level}M.08`
40 }
41
42 let level = forceNumber(videoStream.level).toString(16)
43 if (level.length === 1) level = `0${level}`
44
45 // Default, h264 codec
46 return `${videoCodec}.${baseProfile}${level}`
47}
48
49export async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) {
50 const { audioStream } = await getAudioStream(path, existingProbe)
51
52 if (!audioStream) return ''
53
54 const audioCodecName = audioStream.codec_name
55
56 if (audioCodecName === 'opus') return 'opus'
57 if (audioCodecName === 'vorbis') return 'vorbis'
58 if (audioCodecName === 'aac') return 'mp4a.40.2'
59 if (audioCodecName === 'mp3') return 'mp4a.40.34'
60
61 logger.warn('Cannot get audio codec of %s.', path, { audioStream })
62
63 return 'mp4a.40.2' // Fallback
64}
diff --git a/server/helpers/ffmpeg/ffmpeg-image.ts b/server/helpers/ffmpeg/ffmpeg-image.ts
deleted file mode 100644
index 0bb0ff2c0..000000000
--- a/server/helpers/ffmpeg/ffmpeg-image.ts
+++ /dev/null
@@ -1,14 +0,0 @@
1import { FFmpegImage } from '@shared/ffmpeg'
2import { getFFmpegCommandWrapperOptions } from './ffmpeg-options'
3
4export function processGIF (options: Parameters<FFmpegImage['processGIF']>[0]) {
5 return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).processGIF(options)
6}
7
8export function generateThumbnailFromVideo (options: Parameters<FFmpegImage['generateThumbnailFromVideo']>[0]) {
9 return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).generateThumbnailFromVideo(options)
10}
11
12export function convertWebPToJPG (options: Parameters<FFmpegImage['convertWebPToJPG']>[0]) {
13 return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).convertWebPToJPG(options)
14}
diff --git a/server/helpers/ffmpeg/ffmpeg-options.ts b/server/helpers/ffmpeg/ffmpeg-options.ts
deleted file mode 100644
index 64d7c4179..000000000
--- a/server/helpers/ffmpeg/ffmpeg-options.ts
+++ /dev/null
@@ -1,45 +0,0 @@
1import { logger } from '@server/helpers/logger'
2import { CONFIG } from '@server/initializers/config'
3import { FFMPEG_NICE } from '@server/initializers/constants'
4import { FFmpegCommandWrapperOptions } from '@shared/ffmpeg'
5import { AvailableEncoders } from '@shared/models'
6
7type CommandType = 'live' | 'vod' | 'thumbnail'
8
9export function getFFmpegCommandWrapperOptions (type: CommandType, availableEncoders?: AvailableEncoders): FFmpegCommandWrapperOptions {
10 return {
11 availableEncoders,
12 profile: getProfile(type),
13
14 niceness: FFMPEG_NICE[type.toUpperCase()],
15 tmpDirectory: CONFIG.STORAGE.TMP_DIR,
16 threads: getThreads(type),
17
18 logger: {
19 debug: logger.debug.bind(logger),
20 info: logger.info.bind(logger),
21 warn: logger.warn.bind(logger),
22 error: logger.error.bind(logger)
23 },
24 lTags: { tags: [ 'ffmpeg' ] }
25 }
26}
27
28// ---------------------------------------------------------------------------
29// Private
30// ---------------------------------------------------------------------------
31
32function getThreads (type: CommandType) {
33 if (type === 'live') return CONFIG.LIVE.TRANSCODING.THREADS
34 if (type === 'vod') return CONFIG.TRANSCODING.THREADS
35
36 // Auto
37 return 0
38}
39
40function getProfile (type: CommandType) {
41 if (type === 'live') return CONFIG.LIVE.TRANSCODING.PROFILE
42 if (type === 'vod') return CONFIG.TRANSCODING.PROFILE
43
44 return undefined
45}
diff --git a/server/helpers/ffmpeg/framerate.ts b/server/helpers/ffmpeg/framerate.ts
deleted file mode 100644
index 18cb0e0e2..000000000
--- a/server/helpers/ffmpeg/framerate.ts
+++ /dev/null
@@ -1,44 +0,0 @@
1import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
2import { VideoResolution } from '@shared/models'
3
4export function computeOutputFPS (options: {
5 inputFPS: number
6 resolution: VideoResolution
7}) {
8 const { resolution } = options
9
10 let fps = options.inputFPS
11
12 if (
13 // On small/medium resolutions, limit FPS
14 resolution !== undefined &&
15 resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN &&
16 fps > VIDEO_TRANSCODING_FPS.AVERAGE
17 ) {
18 // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value
19 fps = getClosestFramerateStandard({ fps, type: 'STANDARD' })
20 }
21
22 // Hard FPS limits
23 if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard({ fps, type: 'HD_STANDARD' })
24
25 if (fps < VIDEO_TRANSCODING_FPS.MIN) {
26 throw new Error(`Cannot compute FPS because ${fps} is lower than our minimum value ${VIDEO_TRANSCODING_FPS.MIN}`)
27 }
28
29 return fps
30}
31
32// ---------------------------------------------------------------------------
33// Private
34// ---------------------------------------------------------------------------
35
36function getClosestFramerateStandard (options: {
37 fps: number
38 type: 'HD_STANDARD' | 'STANDARD'
39}) {
40 const { fps, type } = options
41
42 return VIDEO_TRANSCODING_FPS[type].slice(0)
43 .sort((a, b) => fps % a - fps % b)[0]
44}
diff --git a/server/helpers/ffmpeg/index.ts b/server/helpers/ffmpeg/index.ts
deleted file mode 100644
index bf1c73fb6..000000000
--- a/server/helpers/ffmpeg/index.ts
+++ /dev/null
@@ -1,4 +0,0 @@
1export * from './codecs'
2export * from './ffmpeg-image'
3export * from './ffmpeg-options'
4export * from './framerate'
diff --git a/server/helpers/geo-ip.ts b/server/helpers/geo-ip.ts
deleted file mode 100644
index 9e44d660f..000000000
--- a/server/helpers/geo-ip.ts
+++ /dev/null
@@ -1,78 +0,0 @@
1import { pathExists, writeFile } from 'fs-extra'
2import maxmind, { CountryResponse, Reader } from 'maxmind'
3import { join } from 'path'
4import { CONFIG } from '@server/initializers/config'
5import { logger, loggerTagsFactory } from './logger'
6import { isBinaryResponse, peertubeGot } from './requests'
7
8const lTags = loggerTagsFactory('geo-ip')
9
10const mmbdFilename = 'dbip-country-lite-latest.mmdb'
11const mmdbPath = join(CONFIG.STORAGE.BIN_DIR, mmbdFilename)
12
13export class GeoIP {
14 private static instance: GeoIP
15
16 private reader: Reader<CountryResponse>
17
18 private constructor () {
19 }
20
21 async safeCountryISOLookup (ip: string): Promise<string> {
22 if (CONFIG.GEO_IP.ENABLED === false) return null
23
24 await this.initReaderIfNeeded()
25
26 try {
27 const result = this.reader.get(ip)
28 if (!result) return null
29
30 return result.country.iso_code
31 } catch (err) {
32 logger.error('Cannot get country from IP.', { err })
33
34 return null
35 }
36 }
37
38 async updateDatabase () {
39 if (CONFIG.GEO_IP.ENABLED === false) return
40
41 const url = CONFIG.GEO_IP.COUNTRY.DATABASE_URL
42
43 logger.info('Updating GeoIP database from %s.', url, lTags())
44
45 const gotOptions = { context: { bodyKBLimit: 200_000 }, responseType: 'buffer' as 'buffer' }
46
47 try {
48 const gotResult = await peertubeGot(url, gotOptions)
49
50 if (!isBinaryResponse(gotResult)) {
51 throw new Error('Not a binary response')
52 }
53
54 await writeFile(mmdbPath, gotResult.body)
55
56 // Reinit reader
57 this.reader = undefined
58
59 logger.info('GeoIP database updated %s.', mmdbPath, lTags())
60 } catch (err) {
61 logger.error('Cannot update GeoIP database from %s.', url, { err, ...lTags() })
62 }
63 }
64
65 private async initReaderIfNeeded () {
66 if (!this.reader) {
67 if (!await pathExists(mmdbPath)) {
68 await this.updateDatabase()
69 }
70
71 this.reader = await maxmind.open(mmdbPath)
72 }
73 }
74
75 static get Instance () {
76 return this.instance || (this.instance = new this())
77 }
78}
diff --git a/server/helpers/image-utils.ts b/server/helpers/image-utils.ts
deleted file mode 100644
index 2a8bb6e6e..000000000
--- a/server/helpers/image-utils.ts
+++ /dev/null
@@ -1,179 +0,0 @@
1import { copy, readFile, remove, rename } from 'fs-extra'
2import Jimp, { read as jimpRead } from 'jimp'
3import { join } from 'path'
4import { ColorActionName } from '@jimp/plugin-color'
5import { getLowercaseExtension } from '@shared/core-utils'
6import { buildUUID } from '@shared/extra-utils'
7import { convertWebPToJPG, generateThumbnailFromVideo, processGIF } from './ffmpeg'
8import { logger, loggerTagsFactory } from './logger'
9
10const lTags = loggerTagsFactory('image-utils')
11
12function generateImageFilename (extension = '.jpg') {
13 return buildUUID() + extension
14}
15
16async function processImage (options: {
17 path: string
18 destination: string
19 newSize: { width: number, height: number }
20 keepOriginal?: boolean // default false
21}) {
22 const { path, destination, newSize, keepOriginal = false } = options
23
24 const extension = getLowercaseExtension(path)
25
26 if (path === destination) {
27 throw new Error('Jimp/FFmpeg needs an input path different that the output path.')
28 }
29
30 logger.debug('Processing image %s to %s.', path, destination)
31
32 // Use FFmpeg to process GIF
33 if (extension === '.gif') {
34 await processGIF({ path, destination, newSize })
35 } else {
36 await jimpProcessor(path, destination, newSize, extension)
37 }
38
39 if (keepOriginal !== true) await remove(path)
40}
41
42async function generateImageFromVideoFile (options: {
43 fromPath: string
44 folder: string
45 imageName: string
46 size: { width: number, height: number }
47}) {
48 const { fromPath, folder, imageName, size } = options
49
50 const pendingImageName = 'pending-' + imageName
51 const pendingImagePath = join(folder, pendingImageName)
52
53 try {
54 await generateThumbnailFromVideo({ fromPath, output: pendingImagePath })
55
56 const destination = join(folder, imageName)
57 await processImage({ path: pendingImagePath, destination, newSize: size })
58 } catch (err) {
59 logger.error('Cannot generate image from video %s.', fromPath, { err, ...lTags() })
60
61 try {
62 await remove(pendingImagePath)
63 } catch (err) {
64 logger.debug('Cannot remove pending image path after generation error.', { err, ...lTags() })
65 }
66
67 throw err
68 }
69}
70
71async function getImageSize (path: string) {
72 const inputBuffer = await readFile(path)
73
74 const image = await jimpRead(inputBuffer)
75
76 return {
77 width: image.getWidth(),
78 height: image.getHeight()
79 }
80}
81
82// ---------------------------------------------------------------------------
83
84export {
85 generateImageFilename,
86 generateImageFromVideoFile,
87
88 processImage,
89
90 getImageSize
91}
92
93// ---------------------------------------------------------------------------
94
95async function jimpProcessor (path: string, destination: string, newSize: { width: number, height: number }, inputExt: string) {
96 let sourceImage: Jimp
97 const inputBuffer = await readFile(path)
98
99 try {
100 sourceImage = await jimpRead(inputBuffer)
101 } catch (err) {
102 logger.debug('Cannot read %s with jimp. Try to convert the image using ffmpeg first.', path, { err })
103
104 const newName = path + '.jpg'
105 await convertWebPToJPG({ path, destination: newName })
106 await rename(newName, path)
107
108 sourceImage = await jimpRead(path)
109 }
110
111 await remove(destination)
112
113 // Optimization if the source file has the appropriate size
114 const outputExt = getLowercaseExtension(destination)
115 if (skipProcessing({ sourceImage, newSize, imageBytes: inputBuffer.byteLength, inputExt, outputExt })) {
116 return copy(path, destination)
117 }
118
119 await autoResize({ sourceImage, newSize, destination })
120}
121
122async function autoResize (options: {
123 sourceImage: Jimp
124 newSize: { width: number, height: number }
125 destination: string
126}) {
127 const { sourceImage, newSize, destination } = options
128
129 // Portrait mode targeting a landscape, apply some effect on the image
130 const sourceIsPortrait = sourceImage.getWidth() < sourceImage.getHeight()
131 const destIsPortraitOrSquare = newSize.width <= newSize.height
132
133 removeExif(sourceImage)
134
135 if (sourceIsPortrait && !destIsPortraitOrSquare) {
136 const baseImage = sourceImage.cloneQuiet().cover(newSize.width, newSize.height)
137 .color([ { apply: ColorActionName.SHADE, params: [ 50 ] } ])
138
139 const topImage = sourceImage.cloneQuiet().contain(newSize.width, newSize.height)
140
141 return write(baseImage.blit(topImage, 0, 0), destination)
142 }
143
144 return write(sourceImage.cover(newSize.width, newSize.height), destination)
145}
146
147function write (image: Jimp, destination: string) {
148 return image.quality(80).writeAsync(destination)
149}
150
151function skipProcessing (options: {
152 sourceImage: Jimp
153 newSize: { width: number, height: number }
154 imageBytes: number
155 inputExt: string
156 outputExt: string
157}) {
158 const { sourceImage, newSize, imageBytes, inputExt, outputExt } = options
159 const { width, height } = newSize
160
161 if (hasExif(sourceImage)) return false
162 if (sourceImage.getWidth() > width || sourceImage.getHeight() > height) return false
163 if (inputExt !== outputExt) return false
164
165 const kB = 1000
166
167 if (height >= 1000) return imageBytes <= 200 * kB
168 if (height >= 500) return imageBytes <= 100 * kB
169
170 return imageBytes <= 15 * kB
171}
172
173function hasExif (image: Jimp) {
174 return !!(image.bitmap as any).exifBuffer
175}
176
177function removeExif (image: Jimp) {
178 (image.bitmap as any).exifBuffer = null
179}
diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts
deleted file mode 100644
index 6649db40f..000000000
--- a/server/helpers/logger.ts
+++ /dev/null
@@ -1,208 +0,0 @@
1import { stat } from 'fs-extra'
2import { join } from 'path'
3import { format as sqlFormat } from 'sql-formatter'
4import { createLogger, format, transports } from 'winston'
5import { FileTransportOptions } from 'winston/lib/winston/transports'
6import { context } from '@opentelemetry/api'
7import { getSpanContext } from '@opentelemetry/api/build/src/trace/context-utils'
8import { omit } from '@shared/core-utils'
9import { CONFIG } from '../initializers/config'
10import { LOG_FILENAME } from '../initializers/constants'
11
12const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
13
14const consoleLoggerFormat = format.printf(info => {
15 let additionalInfos = JSON.stringify(getAdditionalInfo(info), removeCyclicValues(), 2)
16
17 if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = ''
18 else additionalInfos = ' ' + additionalInfos
19
20 if (info.sql) {
21 if (CONFIG.LOG.PRETTIFY_SQL) {
22 additionalInfos += '\n' + sqlFormat(info.sql, {
23 language: 'sql',
24 tabWidth: 2
25 })
26 } else {
27 additionalInfos += ' - ' + info.sql
28 }
29 }
30
31 return `[${info.label}] ${info.timestamp} ${info.level}: ${info.message}${additionalInfos}`
32})
33
34const jsonLoggerFormat = format.printf(info => {
35 return JSON.stringify(info, removeCyclicValues())
36})
37
38const timestampFormatter = format.timestamp({
39 format: 'YYYY-MM-DD HH:mm:ss.SSS'
40})
41const labelFormatter = (suffix?: string) => {
42 return format.label({
43 label: suffix ? `${label} ${suffix}` : label
44 })
45}
46
47const fileLoggerOptions: FileTransportOptions = {
48 filename: join(CONFIG.STORAGE.LOG_DIR, LOG_FILENAME),
49 handleExceptions: true,
50 format: format.combine(
51 format.timestamp(),
52 jsonLoggerFormat
53 )
54}
55
56if (CONFIG.LOG.ROTATION.ENABLED) {
57 fileLoggerOptions.maxsize = CONFIG.LOG.ROTATION.MAX_FILE_SIZE
58 fileLoggerOptions.maxFiles = CONFIG.LOG.ROTATION.MAX_FILES
59}
60
61function buildLogger (labelSuffix?: string) {
62 return createLogger({
63 level: CONFIG.LOG.LEVEL,
64 defaultMeta: {
65 get traceId () { return getSpanContext(context.active())?.traceId },
66 get spanId () { return getSpanContext(context.active())?.spanId },
67 get traceFlags () { return getSpanContext(context.active())?.traceFlags }
68 },
69 format: format.combine(
70 labelFormatter(labelSuffix),
71 format.splat()
72 ),
73 transports: [
74 new transports.File(fileLoggerOptions),
75 new transports.Console({
76 handleExceptions: true,
77 format: format.combine(
78 timestampFormatter,
79 format.colorize(),
80 consoleLoggerFormat
81 )
82 })
83 ],
84 exitOnError: true
85 })
86}
87
88const logger = buildLogger()
89
90// ---------------------------------------------------------------------------
91
92function bunyanLogFactory (level: string) {
93 return function (...params: any[]) {
94 let meta = null
95 let args = [].concat(params)
96
97 if (arguments[0] instanceof Error) {
98 meta = arguments[0].toString()
99 args = Array.prototype.slice.call(arguments, 1)
100 args.push(meta)
101 } else if (typeof (args[0]) !== 'string') {
102 meta = arguments[0]
103 args = Array.prototype.slice.call(arguments, 1)
104 args.push(meta)
105 }
106
107 logger[level].apply(logger, args)
108 }
109}
110
111const bunyanLogger = {
112 level: () => { },
113 trace: bunyanLogFactory('debug'),
114 debug: bunyanLogFactory('debug'),
115 verbose: bunyanLogFactory('debug'),
116 info: bunyanLogFactory('info'),
117 warn: bunyanLogFactory('warn'),
118 error: bunyanLogFactory('error'),
119 fatal: bunyanLogFactory('error')
120}
121
122// ---------------------------------------------------------------------------
123
124type LoggerTagsFn = (...tags: string[]) => { tags: string[] }
125function loggerTagsFactory (...defaultTags: string[]): LoggerTagsFn {
126 return (...tags: string[]) => {
127 return { tags: defaultTags.concat(tags) }
128 }
129}
130
131// ---------------------------------------------------------------------------
132
133async function mtimeSortFilesDesc (files: string[], basePath: string) {
134 const promises = []
135 const out: { file: string, mtime: number }[] = []
136
137 for (const file of files) {
138 const p = stat(basePath + '/' + file)
139 .then(stats => {
140 if (stats.isFile()) out.push({ file, mtime: stats.mtime.getTime() })
141 })
142
143 promises.push(p)
144 }
145
146 await Promise.all(promises)
147
148 out.sort((a, b) => b.mtime - a.mtime)
149
150 return out
151}
152
153// ---------------------------------------------------------------------------
154
155export {
156 LoggerTagsFn,
157
158 buildLogger,
159 timestampFormatter,
160 labelFormatter,
161 consoleLoggerFormat,
162 jsonLoggerFormat,
163 mtimeSortFilesDesc,
164 logger,
165 loggerTagsFactory,
166 bunyanLogger
167}
168
169// ---------------------------------------------------------------------------
170
171function removeCyclicValues () {
172 const seen = new WeakSet()
173
174 // Thanks: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Cyclic_object_value#Examples
175 return (key: string, value: any) => {
176 if (key === 'cert') return 'Replaced by the logger to avoid large log message'
177
178 if (typeof value === 'object' && value !== null) {
179 if (seen.has(value)) return
180
181 seen.add(value)
182 }
183
184 if (value instanceof Set) {
185 return Array.from(value)
186 }
187
188 if (value instanceof Map) {
189 return Array.from(value.entries())
190 }
191
192 if (value instanceof Error) {
193 const error = {}
194
195 Object.getOwnPropertyNames(value).forEach(key => { error[key] = value[key] })
196
197 return error
198 }
199
200 return value
201 }
202}
203
204function getAdditionalInfo (info: any) {
205 const toOmit = [ 'label', 'timestamp', 'level', 'message', 'sql', 'tags' ]
206
207 return omit(info, toOmit)
208}
diff --git a/server/helpers/markdown.ts b/server/helpers/markdown.ts
deleted file mode 100644
index a20ac22d4..000000000
--- a/server/helpers/markdown.ts
+++ /dev/null
@@ -1,90 +0,0 @@
1import { getDefaultSanitizeOptions, getTextOnlySanitizeOptions, TEXT_WITH_HTML_RULES } from '@shared/core-utils'
2
3const defaultSanitizeOptions = getDefaultSanitizeOptions()
4const textOnlySanitizeOptions = getTextOnlySanitizeOptions()
5
6const sanitizeHtml = require('sanitize-html')
7const markdownItEmoji = require('markdown-it-emoji/light')
8const MarkdownItClass = require('markdown-it')
9
10const markdownItForSafeHtml = new MarkdownItClass('default', { linkify: true, breaks: true, html: true })
11 .enable(TEXT_WITH_HTML_RULES)
12 .use(markdownItEmoji)
13
14const markdownItForPlainText = new MarkdownItClass('default', { linkify: false, breaks: true, html: false })
15 .use(markdownItEmoji)
16 .use(plainTextPlugin)
17
18const toSafeHtml = (text: string) => {
19 if (!text) return ''
20
21 // Restore line feed
22 const textWithLineFeed = text.replace(/<br.?\/?>/g, '\r\n')
23
24 // Convert possible markdown (emojis, emphasis and lists) to html
25 const html = markdownItForSafeHtml.render(textWithLineFeed)
26
27 // Convert to safe Html
28 return sanitizeHtml(html, defaultSanitizeOptions)
29}
30
31const mdToOneLinePlainText = (text: string) => {
32 if (!text) return ''
33
34 markdownItForPlainText.render(text)
35
36 // Convert to safe Html
37 return sanitizeHtml(markdownItForPlainText.plainText, textOnlySanitizeOptions)
38}
39
40// ---------------------------------------------------------------------------
41
42export {
43 toSafeHtml,
44 mdToOneLinePlainText
45}
46
47// ---------------------------------------------------------------------------
48
49// Thanks: https://github.com/wavesheep/markdown-it-plain-text
50function plainTextPlugin (markdownIt: any) {
51 function plainTextRule (state: any) {
52 const text = scan(state.tokens)
53
54 markdownIt.plainText = text
55 }
56
57 function scan (tokens: any[]) {
58 let lastSeparator = ''
59 let text = ''
60
61 function buildSeparator (token: any) {
62 if (token.type === 'list_item_close') {
63 lastSeparator = ', '
64 }
65
66 if (token.tag === 'br' || token.type === 'paragraph_close') {
67 lastSeparator = ' '
68 }
69 }
70
71 for (const token of tokens) {
72 buildSeparator(token)
73
74 if (token.type !== 'inline') continue
75
76 for (const child of token.children) {
77 buildSeparator(child)
78
79 if (!child.content) continue
80
81 text += lastSeparator + child.content
82 lastSeparator = ''
83 }
84 }
85
86 return text
87 }
88
89 markdownIt.core.ruler.push('plainText', plainTextRule)
90}
diff --git a/server/helpers/memoize.ts b/server/helpers/memoize.ts
deleted file mode 100644
index aa20e7d73..000000000
--- a/server/helpers/memoize.ts
+++ /dev/null
@@ -1,12 +0,0 @@
1import memoizee from 'memoizee'
2
3export function Memoize (config?: memoizee.Options<any>) {
4 return function (_target, _key, descriptor: PropertyDescriptor) {
5 const oldFunction = descriptor.value
6 const newFunction = memoizee(oldFunction, config)
7
8 descriptor.value = function () {
9 return newFunction.apply(this, arguments)
10 }
11 }
12}
diff --git a/server/helpers/otp.ts b/server/helpers/otp.ts
deleted file mode 100644
index a32cc9621..000000000
--- a/server/helpers/otp.ts
+++ /dev/null
@@ -1,58 +0,0 @@
1import { Secret, TOTP } from 'otpauth'
2import { CONFIG } from '@server/initializers/config'
3import { WEBSERVER } from '@server/initializers/constants'
4import { decrypt } from './peertube-crypto'
5
6async function isOTPValid (options: {
7 encryptedSecret: string
8 token: string
9}) {
10 const { token, encryptedSecret } = options
11
12 const secret = await decrypt(encryptedSecret, CONFIG.SECRETS.PEERTUBE)
13
14 const totp = new TOTP({
15 ...baseOTPOptions(),
16
17 secret
18 })
19
20 const delta = totp.validate({
21 token,
22 window: 1
23 })
24
25 if (delta === null) return false
26
27 return true
28}
29
30function generateOTPSecret (email: string) {
31 const totp = new TOTP({
32 ...baseOTPOptions(),
33
34 label: email,
35 secret: new Secret()
36 })
37
38 return {
39 secret: totp.secret.base32,
40 uri: totp.toString()
41 }
42}
43
44export {
45 isOTPValid,
46 generateOTPSecret
47}
48
49// ---------------------------------------------------------------------------
50
51function baseOTPOptions () {
52 return {
53 issuer: WEBSERVER.HOST,
54 algorithm: 'SHA1',
55 digits: 6,
56 period: 30
57 }
58}
diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts
deleted file mode 100644
index 95e78a904..000000000
--- a/server/helpers/peertube-crypto.ts
+++ /dev/null
@@ -1,208 +0,0 @@
1import { compare, genSalt, hash } from 'bcrypt'
2import { createCipheriv, createDecipheriv, createSign, createVerify } from 'crypto'
3import { Request } from 'express'
4import { cloneDeep } from 'lodash'
5import { promisify1, promisify2 } from '@shared/core-utils'
6import { sha256 } from '@shared/extra-utils'
7import { BCRYPT_SALT_SIZE, ENCRYPTION, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants'
8import { MActor } from '../types/models'
9import { generateRSAKeyPairPromise, randomBytesPromise, scryptPromise } from './core-utils'
10import { jsonld } from './custom-jsonld-signature'
11import { logger } from './logger'
12
13const bcryptComparePromise = promisify2<any, string, boolean>(compare)
14const bcryptGenSaltPromise = promisify1<number, string>(genSalt)
15const bcryptHashPromise = promisify2<any, string | number, string>(hash)
16
17const httpSignature = require('@peertube/http-signature')
18
19function createPrivateAndPublicKeys () {
20 logger.info('Generating a RSA key...')
21
22 return generateRSAKeyPairPromise(PRIVATE_RSA_KEY_SIZE)
23}
24
25// ---------------------------------------------------------------------------
26// User password checks
27// ---------------------------------------------------------------------------
28
29function comparePassword (plainPassword: string, hashPassword: string) {
30 if (!plainPassword) return Promise.resolve(false)
31
32 return bcryptComparePromise(plainPassword, hashPassword)
33}
34
35async function cryptPassword (password: string) {
36 const salt = await bcryptGenSaltPromise(BCRYPT_SALT_SIZE)
37
38 return bcryptHashPromise(password, salt)
39}
40
41// ---------------------------------------------------------------------------
42// HTTP Signature
43// ---------------------------------------------------------------------------
44
45function isHTTPSignatureDigestValid (rawBody: Buffer, req: Request): boolean {
46 if (req.headers[HTTP_SIGNATURE.HEADER_NAME] && req.headers['digest']) {
47 return buildDigest(rawBody.toString()) === req.headers['digest']
48 }
49
50 return true
51}
52
53function isHTTPSignatureVerified (httpSignatureParsed: any, actor: MActor): boolean {
54 return httpSignature.verifySignature(httpSignatureParsed, actor.publicKey) === true
55}
56
57function parseHTTPSignature (req: Request, clockSkew?: number) {
58 const requiredHeaders = req.method === 'POST'
59 ? [ '(request-target)', 'host', 'digest' ]
60 : [ '(request-target)', 'host' ]
61
62 const parsed = httpSignature.parse(req, { clockSkew, headers: requiredHeaders })
63
64 const parsedHeaders = parsed.params.headers
65 if (!parsedHeaders.includes('date') && !parsedHeaders.includes('(created)')) {
66 throw new Error(`date or (created) must be included in signature`)
67 }
68
69 return parsed
70}
71
72// ---------------------------------------------------------------------------
73// JSONLD
74// ---------------------------------------------------------------------------
75
76function isJsonLDSignatureVerified (fromActor: MActor, signedDocument: any): Promise<boolean> {
77 if (signedDocument.signature.type === 'RsaSignature2017') {
78 return isJsonLDRSA2017Verified(fromActor, signedDocument)
79 }
80
81 logger.warn('Unknown JSON LD signature %s.', signedDocument.signature.type, signedDocument)
82
83 return Promise.resolve(false)
84}
85
86// Backward compatibility with "other" implementations
87async function isJsonLDRSA2017Verified (fromActor: MActor, signedDocument: any) {
88 const [ documentHash, optionsHash ] = await Promise.all([
89 createDocWithoutSignatureHash(signedDocument),
90 createSignatureHash(signedDocument.signature)
91 ])
92
93 const toVerify = optionsHash + documentHash
94
95 const verify = createVerify('RSA-SHA256')
96 verify.update(toVerify, 'utf8')
97
98 return verify.verify(fromActor.publicKey, signedDocument.signature.signatureValue, 'base64')
99}
100
101async function signJsonLDObject <T> (byActor: MActor, data: T) {
102 const signature = {
103 type: 'RsaSignature2017',
104 creator: byActor.url,
105 created: new Date().toISOString()
106 }
107
108 const [ documentHash, optionsHash ] = await Promise.all([
109 createDocWithoutSignatureHash(data),
110 createSignatureHash(signature)
111 ])
112
113 const toSign = optionsHash + documentHash
114
115 const sign = createSign('RSA-SHA256')
116 sign.update(toSign, 'utf8')
117
118 const signatureValue = sign.sign(byActor.privateKey, 'base64')
119 Object.assign(signature, { signatureValue })
120
121 return Object.assign(data, { signature })
122}
123
124// ---------------------------------------------------------------------------
125
126function buildDigest (body: any) {
127 const rawBody = typeof body === 'string' ? body : JSON.stringify(body)
128
129 return 'SHA-256=' + sha256(rawBody, 'base64')
130}
131
132// ---------------------------------------------------------------------------
133// Encryption
134// ---------------------------------------------------------------------------
135
136async function encrypt (str: string, secret: string) {
137 const iv = await randomBytesPromise(ENCRYPTION.IV)
138
139 const key = await scryptPromise(secret, ENCRYPTION.SALT, 32)
140 const cipher = createCipheriv(ENCRYPTION.ALGORITHM, key, iv)
141
142 let encrypted = iv.toString(ENCRYPTION.ENCODING) + ':'
143 encrypted += cipher.update(str, 'utf8', ENCRYPTION.ENCODING)
144 encrypted += cipher.final(ENCRYPTION.ENCODING)
145
146 return encrypted
147}
148
149async function decrypt (encryptedArg: string, secret: string) {
150 const [ ivStr, encryptedStr ] = encryptedArg.split(':')
151
152 const iv = Buffer.from(ivStr, 'hex')
153 const key = await scryptPromise(secret, ENCRYPTION.SALT, 32)
154
155 const decipher = createDecipheriv(ENCRYPTION.ALGORITHM, key, iv)
156
157 return decipher.update(encryptedStr, ENCRYPTION.ENCODING, 'utf8') + decipher.final('utf8')
158}
159
160// ---------------------------------------------------------------------------
161
162export {
163 isHTTPSignatureDigestValid,
164 parseHTTPSignature,
165 isHTTPSignatureVerified,
166 buildDigest,
167 isJsonLDSignatureVerified,
168 comparePassword,
169 createPrivateAndPublicKeys,
170 cryptPassword,
171 signJsonLDObject,
172
173 encrypt,
174 decrypt
175}
176
177// ---------------------------------------------------------------------------
178
179function hashObject (obj: any): Promise<any> {
180 return jsonld.promises.normalize(obj, {
181 safe: false,
182 algorithm: 'URDNA2015',
183 format: 'application/n-quads'
184 }).then(res => sha256(res))
185}
186
187function createSignatureHash (signature: any) {
188 const signatureCopy = cloneDeep(signature)
189 Object.assign(signatureCopy, {
190 '@context': [
191 'https://w3id.org/security/v1',
192 { RsaSignature2017: 'https://w3id.org/security#RsaSignature2017' }
193 ]
194 })
195
196 delete signatureCopy.type
197 delete signatureCopy.id
198 delete signatureCopy.signatureValue
199
200 return hashObject(signatureCopy)
201}
202
203function createDocWithoutSignatureHash (doc: any) {
204 const docWithoutSignature = cloneDeep(doc)
205 delete docWithoutSignature.signature
206
207 return hashObject(docWithoutSignature)
208}
diff --git a/server/helpers/promise-cache.ts b/server/helpers/promise-cache.ts
deleted file mode 100644
index 303bab976..000000000
--- a/server/helpers/promise-cache.ts
+++ /dev/null
@@ -1,39 +0,0 @@
1export class CachePromiseFactory <A, R> {
2 private readonly running = new Map<string, Promise<R>>()
3
4 constructor (
5 private readonly fn: (arg: A) => Promise<R>,
6 private readonly keyBuilder: (arg: A) => string
7 ) {
8 }
9
10 run (arg: A) {
11 return this.runWithContext(null, arg)
12 }
13
14 runWithContext (ctx: any, arg: A) {
15 const key = this.keyBuilder(arg)
16
17 if (this.running.has(key)) return this.running.get(key)
18
19 const p = this.fn.apply(ctx || this, [ arg ])
20
21 this.running.set(key, p)
22
23 return p.finally(() => this.running.delete(key))
24 }
25}
26
27export function CachePromise (options: {
28 keyBuilder: (...args: any[]) => string
29}) {
30 return function (_target, _key, descriptor: PropertyDescriptor) {
31 const promiseCache = new CachePromiseFactory(descriptor.value, options.keyBuilder)
32
33 descriptor.value = function () {
34 if (arguments.length !== 1) throw new Error('Cache promise only support methods with 1 argument')
35
36 return promiseCache.runWithContext(this, arguments[0])
37 }
38 }
39}
diff --git a/server/helpers/proxy.ts b/server/helpers/proxy.ts
deleted file mode 100644
index 8b82ccae0..000000000
--- a/server/helpers/proxy.ts
+++ /dev/null
@@ -1,14 +0,0 @@
1function getProxy () {
2 return process.env.HTTPS_PROXY ||
3 process.env.HTTP_PROXY ||
4 undefined
5}
6
7function isProxyEnabled () {
8 return !!getProxy()
9}
10
11export {
12 getProxy,
13 isProxyEnabled
14}
diff --git a/server/helpers/query.ts b/server/helpers/query.ts
deleted file mode 100644
index c0f78368f..000000000
--- a/server/helpers/query.ts
+++ /dev/null
@@ -1,81 +0,0 @@
1import { pick } from '@shared/core-utils'
2import {
3 VideoChannelsSearchQueryAfterSanitize,
4 VideoPlaylistsSearchQueryAfterSanitize,
5 VideosCommonQueryAfterSanitize,
6 VideosSearchQueryAfterSanitize
7} from '@shared/models'
8
9function pickCommonVideoQuery (query: VideosCommonQueryAfterSanitize) {
10 return pick(query, [
11 'start',
12 'count',
13 'sort',
14 'nsfw',
15 'isLive',
16 'categoryOneOf',
17 'licenceOneOf',
18 'languageOneOf',
19 'privacyOneOf',
20 'tagsOneOf',
21 'tagsAllOf',
22 'isLocal',
23 'include',
24 'skipCount',
25 'hasHLSFiles',
26 'hasWebtorrentFiles', // TODO: Remove in v7
27 'hasWebVideoFiles',
28 'search',
29 'excludeAlreadyWatched'
30 ])
31}
32
33function pickSearchVideoQuery (query: VideosSearchQueryAfterSanitize) {
34 return {
35 ...pickCommonVideoQuery(query),
36
37 ...pick(query, [
38 'searchTarget',
39 'host',
40 'startDate',
41 'endDate',
42 'originallyPublishedStartDate',
43 'originallyPublishedEndDate',
44 'durationMin',
45 'durationMax',
46 'uuids',
47 'excludeAlreadyWatched'
48 ])
49 }
50}
51
52function pickSearchChannelQuery (query: VideoChannelsSearchQueryAfterSanitize) {
53 return pick(query, [
54 'searchTarget',
55 'search',
56 'start',
57 'count',
58 'sort',
59 'host',
60 'handles'
61 ])
62}
63
64function pickSearchPlaylistQuery (query: VideoPlaylistsSearchQueryAfterSanitize) {
65 return pick(query, [
66 'searchTarget',
67 'search',
68 'start',
69 'count',
70 'sort',
71 'host',
72 'uuids'
73 ])
74}
75
76export {
77 pickCommonVideoQuery,
78 pickSearchVideoQuery,
79 pickSearchPlaylistQuery,
80 pickSearchChannelQuery
81}
diff --git a/server/helpers/regexp.ts b/server/helpers/regexp.ts
deleted file mode 100644
index 257054cea..000000000
--- a/server/helpers/regexp.ts
+++ /dev/null
@@ -1,22 +0,0 @@
1// Thanks to https://regex101.com
2function regexpCapture (str: string, regex: RegExp, maxIterations = 100) {
3 const result: RegExpExecArray[] = []
4 let m: RegExpExecArray
5 let i = 0
6
7 while ((m = regex.exec(str)) !== null && i < maxIterations) {
8 // This is necessary to avoid infinite loops with zero-width matches
9 if (m.index === regex.lastIndex) {
10 regex.lastIndex++
11 }
12
13 result.push(m)
14 i++
15 }
16
17 return result
18}
19
20export {
21 regexpCapture
22}
diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts
deleted file mode 100644
index 1625d6e49..000000000
--- a/server/helpers/requests.ts
+++ /dev/null
@@ -1,246 +0,0 @@
1import { createWriteStream, remove } from 'fs-extra'
2import got, { CancelableRequest, NormalizedOptions, Options as GotOptions, RequestError, Response } from 'got'
3import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'
4import { ACTIVITY_PUB, BINARY_CONTENT_TYPES, PEERTUBE_VERSION, REQUEST_TIMEOUTS, WEBSERVER } from '../initializers/constants'
5import { pipelinePromise } from './core-utils'
6import { logger, loggerTagsFactory } from './logger'
7import { getProxy, isProxyEnabled } from './proxy'
8
9const lTags = loggerTagsFactory('request')
10
11const httpSignature = require('@peertube/http-signature')
12
13export interface PeerTubeRequestError extends Error {
14 statusCode?: number
15 responseBody?: any
16 responseHeaders?: any
17 requestHeaders?: any
18}
19
20type PeerTubeRequestOptions = {
21 timeout?: number
22 activityPub?: boolean
23 bodyKBLimit?: number // 1MB
24
25 httpSignature?: {
26 algorithm: string
27 authorizationHeaderName: string
28 keyId: string
29 key: string
30 headers: string[]
31 }
32
33 jsonResponse?: boolean
34
35 followRedirect?: boolean
36} & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
37
38const peertubeGot = got.extend({
39 ...getAgent(),
40
41 headers: {
42 'user-agent': getUserAgent()
43 },
44
45 handlers: [
46 (options, next) => {
47 const promiseOrStream = next(options) as CancelableRequest<any>
48 const bodyKBLimit = options.context?.bodyKBLimit as number
49 if (!bodyKBLimit) throw new Error('No KB limit for this request')
50
51 const bodyLimit = bodyKBLimit * 1000
52
53 /* eslint-disable @typescript-eslint/no-floating-promises */
54 promiseOrStream.on('downloadProgress', progress => {
55 if (progress.transferred > bodyLimit && progress.percent !== 1) {
56 const message = `Exceeded the download limit of ${bodyLimit} B`
57 logger.warn(message, lTags())
58
59 // CancelableRequest
60 if (promiseOrStream.cancel) {
61 promiseOrStream.cancel()
62 return
63 }
64
65 // Stream
66 (promiseOrStream as any).destroy()
67 }
68 })
69
70 return promiseOrStream
71 }
72 ],
73
74 hooks: {
75 beforeRequest: [
76 options => {
77 const headers = options.headers || {}
78 headers['host'] = options.url.host
79 },
80
81 options => {
82 const httpSignatureOptions = options.context?.httpSignature
83
84 if (httpSignatureOptions) {
85 const method = options.method ?? 'GET'
86 const path = options.path ?? options.url.pathname
87
88 if (!method || !path) {
89 throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`)
90 }
91
92 httpSignature.signRequest({
93 getHeader: function (header: string) {
94 const value = options.headers[header.toLowerCase()]
95
96 if (!value) logger.warn('Unknown header requested by http-signature.', { headers: options.headers, header })
97 return value
98 },
99
100 setHeader: function (header: string, value: string) {
101 options.headers[header] = value
102 },
103
104 method,
105 path
106 }, httpSignatureOptions)
107 }
108 }
109 ],
110
111 beforeRetry: [
112 (_options: NormalizedOptions, error: RequestError, retryCount: number) => {
113 logger.debug('Retrying request to %s.', error.request.requestUrl, { retryCount, error: buildRequestError(error), ...lTags() })
114 }
115 ]
116 }
117})
118
119function doRequest (url: string, options: PeerTubeRequestOptions = {}) {
120 const gotOptions = buildGotOptions(options)
121
122 return peertubeGot(url, gotOptions)
123 .catch(err => { throw buildRequestError(err) })
124}
125
126function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) {
127 const gotOptions = buildGotOptions(options)
128
129 return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' })
130 .catch(err => { throw buildRequestError(err) })
131}
132
133async function doRequestAndSaveToFile (
134 url: string,
135 destPath: string,
136 options: PeerTubeRequestOptions = {}
137) {
138 const gotOptions = buildGotOptions({ ...options, timeout: options.timeout ?? REQUEST_TIMEOUTS.FILE })
139
140 const outFile = createWriteStream(destPath)
141
142 try {
143 await pipelinePromise(
144 peertubeGot.stream(url, gotOptions),
145 outFile
146 )
147 } catch (err) {
148 remove(destPath)
149 .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err, ...lTags() }))
150
151 throw buildRequestError(err)
152 }
153}
154
155function getAgent () {
156 if (!isProxyEnabled()) return {}
157
158 const proxy = getProxy()
159
160 logger.info('Using proxy %s.', proxy, lTags())
161
162 const proxyAgentOptions = {
163 keepAlive: true,
164 keepAliveMsecs: 1000,
165 maxSockets: 256,
166 maxFreeSockets: 256,
167 scheduling: 'lifo' as 'lifo',
168 proxy
169 }
170
171 return {
172 agent: {
173 http: new HttpProxyAgent(proxyAgentOptions),
174 https: new HttpsProxyAgent(proxyAgentOptions)
175 }
176 }
177}
178
179function getUserAgent () {
180 return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})`
181}
182
183function isBinaryResponse (result: Response<any>) {
184 return BINARY_CONTENT_TYPES.has(result.headers['content-type'])
185}
186
187// ---------------------------------------------------------------------------
188
189export {
190 PeerTubeRequestOptions,
191
192 doRequest,
193 doJSONRequest,
194 doRequestAndSaveToFile,
195 isBinaryResponse,
196 getAgent,
197 peertubeGot
198}
199
200// ---------------------------------------------------------------------------
201
202function buildGotOptions (options: PeerTubeRequestOptions) {
203 const { activityPub, bodyKBLimit = 1000 } = options
204
205 const context = { bodyKBLimit, httpSignature: options.httpSignature }
206
207 let headers = options.headers || {}
208
209 if (!headers.date) {
210 headers = { ...headers, date: new Date().toUTCString() }
211 }
212
213 if (activityPub && !headers.accept) {
214 headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER }
215 }
216
217 return {
218 method: options.method,
219 dnsCache: true,
220 timeout: options.timeout ?? REQUEST_TIMEOUTS.DEFAULT,
221 json: options.json,
222 searchParams: options.searchParams,
223 followRedirect: options.followRedirect,
224 retry: 2,
225 headers,
226 context
227 }
228}
229
230function buildRequestError (error: RequestError) {
231 const newError: PeerTubeRequestError = new Error(error.message)
232 newError.name = error.name
233 newError.stack = error.stack
234
235 if (error.response) {
236 newError.responseBody = error.response.body
237 newError.responseHeaders = error.response.headers
238 newError.statusCode = error.response.statusCode
239 }
240
241 if (error.options) {
242 newError.requestHeaders = error.options.headers
243 }
244
245 return newError
246}
diff --git a/server/helpers/stream-replacer.ts b/server/helpers/stream-replacer.ts
deleted file mode 100644
index 4babab418..000000000
--- a/server/helpers/stream-replacer.ts
+++ /dev/null
@@ -1,58 +0,0 @@
1import { Transform, TransformCallback } from 'stream'
2
3// Thanks: https://stackoverflow.com/a/45126242
4class StreamReplacer extends Transform {
5 private pendingChunk: Buffer
6
7 constructor (private readonly replacer: (line: string) => string) {
8 super()
9 }
10
11 _transform (chunk: Buffer, _encoding: BufferEncoding, done: TransformCallback) {
12 try {
13 this.pendingChunk = this.pendingChunk?.length
14 ? Buffer.concat([ this.pendingChunk, chunk ])
15 : chunk
16
17 let index: number
18
19 // As long as we keep finding newlines, keep making slices of the buffer and push them to the
20 // readable side of the transform stream
21 while ((index = this.pendingChunk.indexOf('\n')) !== -1) {
22 // The `end` parameter is non-inclusive, so increase it to include the newline we found
23 const line = this.pendingChunk.slice(0, ++index)
24
25 // `start` is inclusive, but we are already one char ahead of the newline -> all good
26 this.pendingChunk = this.pendingChunk.slice(index)
27
28 // We have a single line here! Prepend the string we want
29 this.push(this.doReplace(line))
30 }
31
32 return done()
33 } catch (err) {
34 return done(err)
35 }
36 }
37
38 _flush (done: TransformCallback) {
39 // If we have any remaining data in the cache, send it out
40 if (!this.pendingChunk?.length) return done()
41
42 try {
43 return done(null, this.doReplace(this.pendingChunk))
44 } catch (err) {
45 return done(err)
46 }
47 }
48
49 private doReplace (buffer: Buffer) {
50 const line = this.replacer(buffer.toString('utf8'))
51
52 return Buffer.from(line, 'utf8')
53 }
54}
55
56export {
57 StreamReplacer
58}
diff --git a/server/helpers/token-generator.ts b/server/helpers/token-generator.ts
deleted file mode 100644
index 16313b818..000000000
--- a/server/helpers/token-generator.ts
+++ /dev/null
@@ -1,19 +0,0 @@
1import { buildUUID } from '@shared/extra-utils'
2
3function generateRunnerRegistrationToken () {
4 return 'ptrrt-' + buildUUID()
5}
6
7function generateRunnerToken () {
8 return 'ptrt-' + buildUUID()
9}
10
11function generateRunnerJobToken () {
12 return 'ptrjt-' + buildUUID()
13}
14
15export {
16 generateRunnerRegistrationToken,
17 generateRunnerToken,
18 generateRunnerJobToken
19}
diff --git a/server/helpers/upload.ts b/server/helpers/upload.ts
deleted file mode 100644
index f5f476913..000000000
--- a/server/helpers/upload.ts
+++ /dev/null
@@ -1,14 +0,0 @@
1import { join } from 'path'
2import { DIRECTORIES } from '@server/initializers/constants'
3
4function getResumableUploadPath (filename?: string) {
5 if (filename) return join(DIRECTORIES.RESUMABLE_UPLOAD, filename)
6
7 return DIRECTORIES.RESUMABLE_UPLOAD
8}
9
10// ---------------------------------------------------------------------------
11
12export {
13 getResumableUploadPath
14}
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts
deleted file mode 100644
index 5a4fe4fdd..000000000
--- a/server/helpers/utils.ts
+++ /dev/null
@@ -1,70 +0,0 @@
1import { remove } from 'fs-extra'
2import { Instance as ParseTorrent } from 'parse-torrent'
3import { join } from 'path'
4import { sha256 } from '@shared/extra-utils'
5import { ResultList } from '@shared/models'
6import { CONFIG } from '../initializers/config'
7import { randomBytesPromise } from './core-utils'
8import { logger } from './logger'
9
10function deleteFileAndCatch (path: string) {
11 remove(path)
12 .catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err }))
13}
14
15async function generateRandomString (size: number) {
16 const raw = await randomBytesPromise(size)
17
18 return raw.toString('hex')
19}
20
21interface FormattableToJSON<U, V> {
22 toFormattedJSON (args?: U): V
23}
24
25function getFormattedObjects<U, V, T extends FormattableToJSON<U, V>> (objects: T[], objectsTotal: number, formattedArg?: U) {
26 const formattedObjects = objects.map(o => o.toFormattedJSON(formattedArg))
27
28 return {
29 total: objectsTotal,
30 data: formattedObjects
31 } as ResultList<V>
32}
33
34function generateVideoImportTmpPath (target: string | ParseTorrent, extension = '.mp4') {
35 const id = typeof target === 'string'
36 ? target
37 : target.infoHash
38
39 const hash = sha256(id)
40 return join(CONFIG.STORAGE.TMP_DIR, `${hash}-import${extension}`)
41}
42
43function getSecureTorrentName (originalName: string) {
44 return sha256(originalName) + '.torrent'
45}
46
47/**
48 * From a filename like "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3.mp4", returns
49 * only the "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3" part. If the filename does
50 * not contain a UUID, returns null.
51 */
52function getUUIDFromFilename (filename: string) {
53 const regex = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/
54 const result = filename.match(regex)
55
56 if (!result || Array.isArray(result) === false) return null
57
58 return result[0]
59}
60
61// ---------------------------------------------------------------------------
62
63export {
64 deleteFileAndCatch,
65 generateRandomString,
66 getFormattedObjects,
67 getSecureTorrentName,
68 generateVideoImportTmpPath,
69 getUUIDFromFilename
70}
diff --git a/server/helpers/version.ts b/server/helpers/version.ts
deleted file mode 100644
index 5b3bf59dd..000000000
--- a/server/helpers/version.ts
+++ /dev/null
@@ -1,36 +0,0 @@
1import { execPromise, execPromise2 } from './core-utils'
2import { logger } from './logger'
3
4async function getServerCommit () {
5 try {
6 const tag = await execPromise2(
7 '[ ! -d .git ] || git name-rev --name-only --tags --no-undefined HEAD 2>/dev/null || true',
8 { stdio: [ 0, 1, 2 ] }
9 )
10
11 if (tag) return tag.replace(/^v/, '')
12 } catch (err) {
13 logger.debug('Cannot get version from git tags.', { err })
14 }
15
16 try {
17 const version = await execPromise('[ ! -d .git ] || git rev-parse --short HEAD')
18
19 if (version) return version.toString().trim()
20 } catch (err) {
21 logger.debug('Cannot get version from git HEAD.', { err })
22 }
23
24 return ''
25}
26
27function getNodeABIVersion () {
28 const version = process.versions.modules
29
30 return parseInt(version)
31}
32
33export {
34 getServerCommit,
35 getNodeABIVersion
36}
diff --git a/server/helpers/video.ts b/server/helpers/video.ts
deleted file mode 100644
index c688ef1e3..000000000
--- a/server/helpers/video.ts
+++ /dev/null
@@ -1,51 +0,0 @@
1import { Response } from 'express'
2import { CONFIG } from '@server/initializers/config'
3import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models'
4import { VideoPrivacy, VideoState } from '@shared/models'
5import { forceNumber } from '@shared/core-utils'
6
7function getVideoWithAttributes (res: Response) {
8 return res.locals.videoAPI || res.locals.videoAll || res.locals.onlyVideo
9}
10
11function extractVideo (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
12 return isStreamingPlaylist(videoOrPlaylist)
13 ? videoOrPlaylist.Video
14 : videoOrPlaylist
15}
16
17function isPrivacyForFederation (privacy: VideoPrivacy) {
18 const castedPrivacy = forceNumber(privacy)
19
20 return castedPrivacy === VideoPrivacy.PUBLIC ||
21 (CONFIG.FEDERATION.VIDEOS.FEDERATE_UNLISTED === true && castedPrivacy === VideoPrivacy.UNLISTED)
22}
23
24function isStateForFederation (state: VideoState) {
25 const castedState = forceNumber(state)
26
27 return castedState === VideoState.PUBLISHED || castedState === VideoState.WAITING_FOR_LIVE || castedState === VideoState.LIVE_ENDED
28}
29
30function getPrivaciesForFederation () {
31 return (CONFIG.FEDERATION.VIDEOS.FEDERATE_UNLISTED === true)
32 ? [ { privacy: VideoPrivacy.PUBLIC }, { privacy: VideoPrivacy.UNLISTED } ]
33 : [ { privacy: VideoPrivacy.PUBLIC } ]
34}
35
36function getExtFromMimetype (mimeTypes: { [id: string]: string | string[] }, mimeType: string) {
37 const value = mimeTypes[mimeType]
38
39 if (Array.isArray(value)) return value[0]
40
41 return value
42}
43
44export {
45 getVideoWithAttributes,
46 extractVideo,
47 getExtFromMimetype,
48 isStateForFederation,
49 isPrivacyForFederation,
50 getPrivaciesForFederation
51}
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
deleted file mode 100644
index f33a7bccd..000000000
--- a/server/helpers/webtorrent.ts
+++ /dev/null
@@ -1,258 +0,0 @@
1import { decode, encode } from 'bencode'
2import createTorrent from 'create-torrent'
3import { createWriteStream, ensureDir, pathExists, readFile, remove, writeFile } from 'fs-extra'
4import { encode as magnetUriEncode } from 'magnet-uri'
5import parseTorrent from 'parse-torrent'
6import { dirname, join } from 'path'
7import { pipeline } from 'stream'
8import WebTorrent, { Instance, TorrentFile } from 'webtorrent'
9import { isArray } from '@server/helpers/custom-validators/misc'
10import { WEBSERVER } from '@server/initializers/constants'
11import { generateTorrentFileName } from '@server/lib/paths'
12import { VideoPathManager } from '@server/lib/video-path-manager'
13import { MVideo } from '@server/types/models/video/video'
14import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file'
15import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist'
16import { promisify2 } from '@shared/core-utils'
17import { sha1 } from '@shared/extra-utils'
18import { CONFIG } from '../initializers/config'
19import { logger } from './logger'
20import { generateVideoImportTmpPath } from './utils'
21import { extractVideo } from './video'
22
23const createTorrentPromise = promisify2<string, any, any>(createTorrent)
24
25async function downloadWebTorrentVideo (target: { uri: string, torrentName?: string }, timeout: number) {
26 const id = target.uri || target.torrentName
27 let timer
28
29 const path = generateVideoImportTmpPath(id)
30 logger.info('Importing torrent video %s', id)
31
32 const directoryPath = join(CONFIG.STORAGE.TMP_DIR, 'webtorrent')
33 await ensureDir(directoryPath)
34
35 return new Promise<string>((res, rej) => {
36 const webtorrent = new WebTorrent()
37 let file: TorrentFile
38
39 const torrentId = target.uri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName)
40
41 const options = { path: directoryPath }
42 const torrent = webtorrent.add(torrentId, options, torrent => {
43 if (torrent.files.length !== 1) {
44 if (timer) clearTimeout(timer)
45
46 for (const file of torrent.files) {
47 deleteDownloadedFile({ directoryPath, filepath: file.path })
48 }
49
50 return safeWebtorrentDestroy(webtorrent, torrentId, undefined, target.torrentName)
51 .then(() => rej(new Error('Cannot import torrent ' + torrentId + ': there are multiple files in it')))
52 }
53
54 logger.debug('Got torrent from webtorrent %s.', id, { infoHash: torrent.infoHash })
55
56 file = torrent.files[0]
57
58 // FIXME: avoid creating another stream when https://github.com/webtorrent/webtorrent/issues/1517 is fixed
59 const writeStream = createWriteStream(path)
60 writeStream.on('finish', () => {
61 if (timer) clearTimeout(timer)
62
63 safeWebtorrentDestroy(webtorrent, torrentId, { directoryPath, filepath: file.path }, target.torrentName)
64 .then(() => res(path))
65 .catch(err => logger.error('Cannot destroy webtorrent.', { err }))
66 })
67
68 pipeline(
69 file.createReadStream(),
70 writeStream,
71 err => {
72 if (err) rej(err)
73 }
74 )
75 })
76
77 torrent.on('error', err => rej(err))
78
79 timer = setTimeout(() => {
80 const err = new Error('Webtorrent download timeout.')
81
82 safeWebtorrentDestroy(webtorrent, torrentId, file ? { directoryPath, filepath: file.path } : undefined, target.torrentName)
83 .then(() => rej(err))
84 .catch(destroyErr => {
85 logger.error('Cannot destroy webtorrent.', { err: destroyErr })
86 rej(err)
87 })
88
89 }, timeout)
90 })
91}
92
93function createTorrentAndSetInfoHash (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
94 return VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(videoOrPlaylist), videoPath => {
95 return createTorrentAndSetInfoHashFromPath(videoOrPlaylist, videoFile, videoPath)
96 })
97}
98
99async function createTorrentAndSetInfoHashFromPath (
100 videoOrPlaylist: MVideo | MStreamingPlaylistVideo,
101 videoFile: MVideoFile,
102 filePath: string
103) {
104 const video = extractVideo(videoOrPlaylist)
105
106 const options = {
107 // Keep the extname, it's used by the client to stream the file inside a web browser
108 name: buildInfoName(video, videoFile),
109 createdBy: 'PeerTube',
110 announceList: buildAnnounceList(),
111 urlList: buildUrlList(video, videoFile)
112 }
113
114 const torrentContent = await createTorrentPromise(filePath, options)
115
116 const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution)
117 const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename)
118 logger.info('Creating torrent %s.', torrentPath)
119
120 await writeFile(torrentPath, torrentContent)
121
122 // Remove old torrent file if it existed
123 if (videoFile.hasTorrent()) {
124 await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename))
125 }
126
127 const parsedTorrent = parseTorrent(torrentContent)
128 videoFile.infoHash = parsedTorrent.infoHash
129 videoFile.torrentFilename = torrentFilename
130}
131
132async function updateTorrentMetadata (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
133 const video = extractVideo(videoOrPlaylist)
134
135 const oldTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)
136
137 if (!await pathExists(oldTorrentPath)) {
138 logger.info('Do not update torrent metadata %s of video %s because the file does not exist anymore.', video.uuid, oldTorrentPath)
139 return
140 }
141
142 const torrentContent = await readFile(oldTorrentPath)
143 const decoded = decode(torrentContent)
144
145 decoded['announce-list'] = buildAnnounceList()
146 decoded.announce = decoded['announce-list'][0][0]
147
148 decoded['url-list'] = buildUrlList(video, videoFile)
149
150 decoded.info.name = buildInfoName(video, videoFile)
151 decoded['creation date'] = Math.ceil(Date.now() / 1000)
152
153 const newTorrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution)
154 const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, newTorrentFilename)
155
156 logger.info('Updating torrent metadata %s -> %s.', oldTorrentPath, newTorrentPath)
157
158 await writeFile(newTorrentPath, encode(decoded))
159 await remove(oldTorrentPath)
160
161 videoFile.torrentFilename = newTorrentFilename
162 videoFile.infoHash = sha1(encode(decoded.info))
163}
164
165function generateMagnetUri (
166 video: MVideo,
167 videoFile: MVideoFileRedundanciesOpt,
168 trackerUrls: string[]
169) {
170 const xs = videoFile.getTorrentUrl()
171 const announce = trackerUrls
172
173 let urlList = video.hasPrivateStaticPath()
174 ? []
175 : [ videoFile.getFileUrl(video) ]
176
177 const redundancies = videoFile.RedundancyVideos
178 if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl))
179
180 const magnetHash = {
181 xs,
182 announce,
183 urlList,
184 infoHash: videoFile.infoHash,
185 name: video.name
186 }
187
188 return magnetUriEncode(magnetHash)
189}
190
191// ---------------------------------------------------------------------------
192
193export {
194 createTorrentPromise,
195 updateTorrentMetadata,
196
197 createTorrentAndSetInfoHash,
198 createTorrentAndSetInfoHashFromPath,
199
200 generateMagnetUri,
201 downloadWebTorrentVideo
202}
203
204// ---------------------------------------------------------------------------
205
206function safeWebtorrentDestroy (
207 webtorrent: Instance,
208 torrentId: string,
209 downloadedFile?: { directoryPath: string, filepath: string },
210 torrentName?: string
211) {
212 return new Promise<void>(res => {
213 webtorrent.destroy(err => {
214 // Delete torrent file
215 if (torrentName) {
216 logger.debug('Removing %s torrent after webtorrent download.', torrentId)
217 remove(torrentId)
218 .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err }))
219 }
220
221 // Delete downloaded file
222 if (downloadedFile) deleteDownloadedFile(downloadedFile)
223
224 if (err) logger.warn('Cannot destroy webtorrent in timeout.', { err })
225
226 return res()
227 })
228 })
229}
230
231function deleteDownloadedFile (downloadedFile: { directoryPath: string, filepath: string }) {
232 // We want to delete the base directory
233 let pathToDelete = dirname(downloadedFile.filepath)
234 if (pathToDelete === '.') pathToDelete = downloadedFile.filepath
235
236 const toRemovePath = join(downloadedFile.directoryPath, pathToDelete)
237
238 logger.debug('Removing %s after webtorrent download.', toRemovePath)
239 remove(toRemovePath)
240 .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', toRemovePath, { err }))
241}
242
243function buildAnnounceList () {
244 return [
245 [ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ],
246 [ WEBSERVER.URL + '/tracker/announce' ]
247 ]
248}
249
250function buildUrlList (video: MVideo, videoFile: MVideoFile) {
251 if (video.hasPrivateStaticPath()) return []
252
253 return [ videoFile.getFileUrl(video) ]
254}
255
256function buildInfoName (video: MVideo, videoFile: MVideoFile) {
257 return `${video.name} ${videoFile.resolution}p${videoFile.extname}`
258}
diff --git a/server/helpers/youtube-dl/index.ts b/server/helpers/youtube-dl/index.ts
deleted file mode 100644
index 6afc77dcf..000000000
--- a/server/helpers/youtube-dl/index.ts
+++ /dev/null
@@ -1,3 +0,0 @@
1export * from './youtube-dl-cli'
2export * from './youtube-dl-info-builder'
3export * from './youtube-dl-wrapper'
diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts
deleted file mode 100644
index 765038cea..000000000
--- a/server/helpers/youtube-dl/youtube-dl-cli.ts
+++ /dev/null
@@ -1,259 +0,0 @@
1import execa from 'execa'
2import { ensureDir, pathExists, writeFile } from 'fs-extra'
3import { dirname, join } from 'path'
4import { CONFIG } from '@server/initializers/config'
5import { VideoResolution } from '@shared/models'
6import { logger, loggerTagsFactory } from '../logger'
7import { getProxy, isProxyEnabled } from '../proxy'
8import { isBinaryResponse, peertubeGot } from '../requests'
9import { OptionsOfBufferResponseBody } from 'got/dist/source'
10
11const lTags = loggerTagsFactory('youtube-dl')
12
13const youtubeDLBinaryPath = join(CONFIG.STORAGE.BIN_DIR, CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME)
14
15export class YoutubeDLCLI {
16
17 static async safeGet () {
18 if (!await pathExists(youtubeDLBinaryPath)) {
19 await ensureDir(dirname(youtubeDLBinaryPath))
20
21 await this.updateYoutubeDLBinary()
22 }
23
24 return new YoutubeDLCLI()
25 }
26
27 static async updateYoutubeDLBinary () {
28 const url = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.URL
29
30 logger.info('Updating youtubeDL binary from %s.', url, lTags())
31
32 const gotOptions: OptionsOfBufferResponseBody = {
33 context: { bodyKBLimit: 20_000 },
34 responseType: 'buffer' as 'buffer'
35 }
36
37 if (process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN) {
38 gotOptions.headers = {
39 authorization: 'Bearer ' + process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN
40 }
41 }
42
43 try {
44 let gotResult = await peertubeGot(url, gotOptions)
45
46 if (!isBinaryResponse(gotResult)) {
47 const json = JSON.parse(gotResult.body.toString())
48 const latest = json.filter(release => release.prerelease === false)[0]
49 if (!latest) throw new Error('Cannot find latest release')
50
51 const releaseName = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME
52 const releaseAsset = latest.assets.find(a => a.name === releaseName)
53 if (!releaseAsset) throw new Error(`Cannot find appropriate release with name ${releaseName} in release assets`)
54
55 gotResult = await peertubeGot(releaseAsset.browser_download_url, gotOptions)
56 }
57
58 if (!isBinaryResponse(gotResult)) {
59 throw new Error('Not a binary response')
60 }
61
62 await writeFile(youtubeDLBinaryPath, gotResult.body)
63
64 logger.info('youtube-dl updated %s.', youtubeDLBinaryPath, lTags())
65 } catch (err) {
66 logger.error('Cannot update youtube-dl from %s.', url, { err, ...lTags() })
67 }
68 }
69
70 static getYoutubeDLVideoFormat (enabledResolutions: VideoResolution[], useBestFormat: boolean) {
71 /**
72 * list of format selectors in order or preference
73 * see https://github.com/ytdl-org/youtube-dl#format-selection
74 *
75 * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope
76 * of being able to do a "quick-transcode"
77 * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9)
78 * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback
79 *
80 * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499
81 **/
82
83 let result: string[] = []
84
85 if (!useBestFormat) {
86 const resolution = enabledResolutions.length === 0
87 ? VideoResolution.H_720P
88 : Math.max(...enabledResolutions)
89
90 result = [
91 `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1
92 `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2
93 `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]` // case #
94 ]
95 }
96
97 return result.concat([
98 'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio',
99 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
100 'bestvideo[ext=mp4]+bestaudio[ext=m4a]',
101 'best' // Ultimate fallback
102 ]).join('/')
103 }
104
105 private constructor () {
106
107 }
108
109 download (options: {
110 url: string
111 format: string
112 output: string
113 processOptions: execa.NodeOptions
114 timeout?: number
115 additionalYoutubeDLArgs?: string[]
116 }) {
117 let args = options.additionalYoutubeDLArgs || []
118 args = args.concat([ '--merge-output-format', 'mp4', '-f', options.format, '-o', options.output ])
119
120 return this.run({
121 url: options.url,
122 processOptions: options.processOptions,
123 timeout: options.timeout,
124 args
125 })
126 }
127
128 async getInfo (options: {
129 url: string
130 format: string
131 processOptions: execa.NodeOptions
132 additionalYoutubeDLArgs?: string[]
133 }) {
134 const { url, format, additionalYoutubeDLArgs = [], processOptions } = options
135
136 const completeArgs = additionalYoutubeDLArgs.concat([ '--dump-json', '-f', format ])
137
138 const data = await this.run({ url, args: completeArgs, processOptions })
139 if (!data) return undefined
140
141 const info = data.map(d => JSON.parse(d))
142
143 return info.length === 1
144 ? info[0]
145 : info
146 }
147
148 async getListInfo (options: {
149 url: string
150 latestVideosCount?: number
151 processOptions: execa.NodeOptions
152 }): Promise<{ upload_date: string, webpage_url: string }[]> {
153 const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
154
155 if (CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME === 'yt-dlp') {
156 // Optimize listing videos only when using yt-dlp because it is bugged with youtube-dl when fetching a channel
157 additionalYoutubeDLArgs.push('--flat-playlist')
158 }
159
160 if (options.latestVideosCount !== undefined) {
161 additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
162 }
163
164 const result = await this.getInfo({
165 url: options.url,
166 format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
167 processOptions: options.processOptions,
168 additionalYoutubeDLArgs
169 })
170
171 if (!result) return result
172 if (!Array.isArray(result)) return [ result ]
173
174 return result
175 }
176
177 async getSubs (options: {
178 url: string
179 format: 'vtt'
180 processOptions: execa.NodeOptions
181 }) {
182 const { url, format, processOptions } = options
183
184 const args = [ '--skip-download', '--all-subs', `--sub-format=${format}` ]
185
186 const data = await this.run({ url, args, processOptions })
187 const files: string[] = []
188
189 const skipString = '[info] Writing video subtitles to: '
190
191 for (let i = 0, len = data.length; i < len; i++) {
192 const line = data[i]
193
194 if (line.indexOf(skipString) === 0) {
195 files.push(line.slice(skipString.length))
196 }
197 }
198
199 return files
200 }
201
202 private async run (options: {
203 url: string
204 args: string[]
205 timeout?: number
206 processOptions: execa.NodeOptions
207 }) {
208 const { url, args, timeout, processOptions } = options
209
210 let completeArgs = this.wrapWithProxyOptions(args)
211 completeArgs = this.wrapWithIPOptions(completeArgs)
212 completeArgs = this.wrapWithFFmpegOptions(completeArgs)
213
214 const { PYTHON_PATH } = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE
215 const subProcess = execa(PYTHON_PATH, [ youtubeDLBinaryPath, ...completeArgs, url ], processOptions)
216
217 if (timeout) {
218 setTimeout(() => subProcess.cancel(), timeout)
219 }
220
221 const output = await subProcess
222
223 logger.debug('Run youtube-dl command.', { command: output.command, ...lTags() })
224
225 return output.stdout
226 ? output.stdout.trim().split(/\r?\n/)
227 : undefined
228 }
229
230 private wrapWithProxyOptions (args: string[]) {
231 if (isProxyEnabled()) {
232 logger.debug('Using proxy %s for YoutubeDL', getProxy(), lTags())
233
234 return [ '--proxy', getProxy() ].concat(args)
235 }
236
237 return args
238 }
239
240 private wrapWithIPOptions (args: string[]) {
241 if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) {
242 logger.debug('Force ipv4 for YoutubeDL')
243
244 return [ '--force-ipv4' ].concat(args)
245 }
246
247 return args
248 }
249
250 private wrapWithFFmpegOptions (args: string[]) {
251 if (process.env.FFMPEG_PATH) {
252 logger.debug('Using ffmpeg location %s for YoutubeDL', process.env.FFMPEG_PATH, lTags())
253
254 return [ '--ffmpeg-location', process.env.FFMPEG_PATH ].concat(args)
255 }
256
257 return args
258 }
259}
diff --git a/server/helpers/youtube-dl/youtube-dl-info-builder.ts b/server/helpers/youtube-dl/youtube-dl-info-builder.ts
deleted file mode 100644
index a74904e43..000000000
--- a/server/helpers/youtube-dl/youtube-dl-info-builder.ts
+++ /dev/null
@@ -1,205 +0,0 @@
1import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../../initializers/constants'
2import { peertubeTruncate } from '../core-utils'
3import { isUrlValid } from '../custom-validators/activitypub/misc'
4
5type YoutubeDLInfo = {
6 name?: string
7 description?: string
8 category?: number
9 language?: string
10 licence?: number
11 nsfw?: boolean
12 tags?: string[]
13 thumbnailUrl?: string
14 ext?: string
15 originallyPublishedAtWithoutTime?: Date
16 webpageUrl?: string
17
18 urls?: string[]
19}
20
21class YoutubeDLInfoBuilder {
22 private readonly info: any
23
24 constructor (info: any) {
25 this.info = { ...info }
26 }
27
28 getInfo () {
29 const obj = this.buildVideoInfo(this.normalizeObject(this.info))
30 if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video'
31
32 return obj
33 }
34
35 private normalizeObject (obj: any) {
36 const newObj: any = {}
37
38 for (const key of Object.keys(obj)) {
39 // Deprecated key
40 if (key === 'resolution') continue
41
42 const value = obj[key]
43
44 if (typeof value === 'string') {
45 newObj[key] = value.normalize()
46 } else {
47 newObj[key] = value
48 }
49 }
50
51 return newObj
52 }
53
54 private buildOriginallyPublishedAt (obj: any) {
55 let originallyPublishedAt: Date = null
56
57 const uploadDateMatcher = /^(\d{4})(\d{2})(\d{2})$/.exec(obj.upload_date)
58 if (uploadDateMatcher) {
59 originallyPublishedAt = new Date()
60 originallyPublishedAt.setHours(0, 0, 0, 0)
61
62 const year = parseInt(uploadDateMatcher[1], 10)
63 // Month starts from 0
64 const month = parseInt(uploadDateMatcher[2], 10) - 1
65 const day = parseInt(uploadDateMatcher[3], 10)
66
67 originallyPublishedAt.setFullYear(year, month, day)
68 }
69
70 return originallyPublishedAt
71 }
72
73 private buildVideoInfo (obj: any): YoutubeDLInfo {
74 return {
75 name: this.titleTruncation(obj.title),
76 description: this.descriptionTruncation(obj.description),
77 category: this.getCategory(obj.categories),
78 licence: this.getLicence(obj.license),
79 language: this.getLanguage(obj.language),
80 nsfw: this.isNSFW(obj),
81 tags: this.getTags(obj.tags),
82 thumbnailUrl: obj.thumbnail || undefined,
83 urls: this.buildAvailableUrl(obj),
84 originallyPublishedAtWithoutTime: this.buildOriginallyPublishedAt(obj),
85 ext: obj.ext,
86 webpageUrl: obj.webpage_url
87 }
88 }
89
90 private buildAvailableUrl (obj: any) {
91 const urls: string[] = []
92
93 if (obj.url) urls.push(obj.url)
94 if (obj.urls) {
95 if (Array.isArray(obj.urls)) urls.push(...obj.urls)
96 else urls.push(obj.urls)
97 }
98
99 const formats = Array.isArray(obj.formats)
100 ? obj.formats
101 : []
102
103 for (const format of formats) {
104 if (!format.url) continue
105
106 urls.push(format.url)
107 }
108
109 const thumbnails = Array.isArray(obj.thumbnails)
110 ? obj.thumbnails
111 : []
112
113 for (const thumbnail of thumbnails) {
114 if (!thumbnail.url) continue
115
116 urls.push(thumbnail.url)
117 }
118
119 if (obj.thumbnail) urls.push(obj.thumbnail)
120
121 for (const subtitleKey of Object.keys(obj.subtitles || {})) {
122 const subtitles = obj.subtitles[subtitleKey]
123 if (!Array.isArray(subtitles)) continue
124
125 for (const subtitle of subtitles) {
126 if (!subtitle.url) continue
127
128 urls.push(subtitle.url)
129 }
130 }
131
132 return urls.filter(u => u && isUrlValid(u))
133 }
134
135 private titleTruncation (title: string) {
136 return peertubeTruncate(title, {
137 length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
138 separator: /,? +/,
139 omission: ' […]'
140 })
141 }
142
143 private descriptionTruncation (description: string) {
144 if (!description || description.length < CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.min) return undefined
145
146 return peertubeTruncate(description, {
147 length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max,
148 separator: /,? +/,
149 omission: ' […]'
150 })
151 }
152
153 private isNSFW (info: any) {
154 return info?.age_limit >= 16
155 }
156
157 private getTags (tags: string[]) {
158 if (Array.isArray(tags) === false) return []
159
160 return tags
161 .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
162 .map(t => t.normalize())
163 .slice(0, 5)
164 }
165
166 private getLicence (licence: string) {
167 if (!licence) return undefined
168
169 if (licence.includes('Creative Commons Attribution')) return 1
170
171 for (const key of Object.keys(VIDEO_LICENCES)) {
172 const peertubeLicence = VIDEO_LICENCES[key]
173 if (peertubeLicence.toLowerCase() === licence.toLowerCase()) return parseInt(key, 10)
174 }
175
176 return undefined
177 }
178
179 private getCategory (categories: string[]) {
180 if (!categories) return undefined
181
182 const categoryString = categories[0]
183 if (!categoryString || typeof categoryString !== 'string') return undefined
184
185 if (categoryString === 'News & Politics') return 11
186
187 for (const key of Object.keys(VIDEO_CATEGORIES)) {
188 const category = VIDEO_CATEGORIES[key]
189 if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10)
190 }
191
192 return undefined
193 }
194
195 private getLanguage (language: string) {
196 return VIDEO_LANGUAGES[language] ? language : undefined
197 }
198}
199
200// ---------------------------------------------------------------------------
201
202export {
203 YoutubeDLInfo,
204 YoutubeDLInfoBuilder
205}
diff --git a/server/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/helpers/youtube-dl/youtube-dl-wrapper.ts
deleted file mode 100644
index ac3cd190e..000000000
--- a/server/helpers/youtube-dl/youtube-dl-wrapper.ts
+++ /dev/null
@@ -1,154 +0,0 @@
1import { move, pathExists, readdir, remove } from 'fs-extra'
2import { dirname, join } from 'path'
3import { inspect } from 'util'
4import { CONFIG } from '@server/initializers/config'
5import { isVideoFileExtnameValid } from '../custom-validators/videos'
6import { logger, loggerTagsFactory } from '../logger'
7import { generateVideoImportTmpPath } from '../utils'
8import { YoutubeDLCLI } from './youtube-dl-cli'
9import { YoutubeDLInfo, YoutubeDLInfoBuilder } from './youtube-dl-info-builder'
10
11const lTags = loggerTagsFactory('youtube-dl')
12
13export type YoutubeDLSubs = {
14 language: string
15 filename: string
16 path: string
17}[]
18
19const processOptions = {
20 maxBuffer: 1024 * 1024 * 30 // 30MB
21}
22
23class YoutubeDLWrapper {
24
25 constructor (
26 private readonly url: string,
27 private readonly enabledResolutions: number[],
28 private readonly useBestFormat: boolean
29 ) {
30
31 }
32
33 async getInfoForDownload (youtubeDLArgs: string[] = []): Promise<YoutubeDLInfo> {
34 const youtubeDL = await YoutubeDLCLI.safeGet()
35
36 const info = await youtubeDL.getInfo({
37 url: this.url,
38 format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat),
39 additionalYoutubeDLArgs: youtubeDLArgs,
40 processOptions
41 })
42
43 if (!info) throw new Error(`YoutubeDL could not get info from ${this.url}`)
44
45 if (info.is_live === true) throw new Error('Cannot download a live streaming.')
46
47 const infoBuilder = new YoutubeDLInfoBuilder(info)
48
49 return infoBuilder.getInfo()
50 }
51
52 async getInfoForListImport (options: {
53 latestVideosCount?: number
54 }) {
55 const youtubeDL = await YoutubeDLCLI.safeGet()
56
57 const list = await youtubeDL.getListInfo({
58 url: this.url,
59 latestVideosCount: options.latestVideosCount,
60 processOptions
61 })
62
63 if (!Array.isArray(list)) throw new Error(`YoutubeDL could not get list info from ${this.url}: ${inspect(list)}`)
64
65 return list.map(info => info.webpage_url)
66 }
67
68 async getSubtitles (): Promise<YoutubeDLSubs> {
69 const cwd = CONFIG.STORAGE.TMP_DIR
70
71 const youtubeDL = await YoutubeDLCLI.safeGet()
72
73 const files = await youtubeDL.getSubs({ url: this.url, format: 'vtt', processOptions: { cwd } })
74 if (!files) return []
75
76 logger.debug('Get subtitles from youtube dl.', { url: this.url, files, ...lTags() })
77
78 const subtitles = files.reduce((acc, filename) => {
79 const matched = filename.match(/\.([a-z]{2})(-[a-z]+)?\.(vtt|ttml)/i)
80 if (!matched?.[1]) return acc
81
82 return [
83 ...acc,
84 {
85 language: matched[1],
86 path: join(cwd, filename),
87 filename
88 }
89 ]
90 }, [])
91
92 return subtitles
93 }
94
95 async downloadVideo (fileExt: string, timeout: number): Promise<string> {
96 // Leave empty the extension, youtube-dl will add it
97 const pathWithoutExtension = generateVideoImportTmpPath(this.url, '')
98
99 logger.info('Importing youtubeDL video %s to %s', this.url, pathWithoutExtension, lTags())
100
101 const youtubeDL = await YoutubeDLCLI.safeGet()
102
103 try {
104 await youtubeDL.download({
105 url: this.url,
106 format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat),
107 output: pathWithoutExtension,
108 timeout,
109 processOptions
110 })
111
112 // If youtube-dl did not guess an extension for our file, just use .mp4 as default
113 if (await pathExists(pathWithoutExtension)) {
114 await move(pathWithoutExtension, pathWithoutExtension + '.mp4')
115 }
116
117 return this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
118 } catch (err) {
119 this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
120 .then(path => {
121 logger.debug('Error in youtube-dl import, deleting file %s.', path, { err, ...lTags() })
122
123 return remove(path)
124 })
125 .catch(innerErr => logger.error('Cannot remove file in youtubeDL error.', { innerErr, ...lTags() }))
126
127 throw err
128 }
129 }
130
131 private async guessVideoPathWithExtension (tmpPath: string, sourceExt: string) {
132 if (!isVideoFileExtnameValid(sourceExt)) {
133 throw new Error('Invalid video extension ' + sourceExt)
134 }
135
136 const extensions = [ sourceExt, '.mp4', '.mkv', '.webm' ]
137
138 for (const extension of extensions) {
139 const path = tmpPath + extension
140
141 if (await pathExists(path)) return path
142 }
143
144 const directoryContent = await readdir(dirname(tmpPath))
145
146 throw new Error(`Cannot guess path of ${tmpPath}. Directory content: ${directoryContent.join(', ')}`)
147 }
148}
149
150// ---------------------------------------------------------------------------
151
152export {
153 YoutubeDLWrapper
154}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 13:35:25 +0000