diff options
Diffstat (limited to 'server/helpers')
82 files changed, 0 insertions, 6151 deletions
diff --git a/server/helpers/actors.ts b/server/helpers/actors.ts deleted file mode 100644 index c31fe6f8e..000000000 --- a/server/helpers/actors.ts +++ /dev/null | |||
@@ -1,17 +0,0 @@ | |||
1 | import { WEBSERVER } from '@server/initializers/constants' | ||
2 | |||
3 | function handleToNameAndHost (handle: string) { | ||
4 | let [ name, host ] = handle.split('@') | ||
5 | if (host === WEBSERVER.HOST) host = null | ||
6 | |||
7 | return { name, host, handle } | ||
8 | } | ||
9 | |||
10 | function handlesToNameAndHost (handles: string[]) { | ||
11 | return handles.map(h => handleToNameAndHost(h)) | ||
12 | } | ||
13 | |||
14 | export { | ||
15 | handleToNameAndHost, | ||
16 | handlesToNameAndHost | ||
17 | } | ||
diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts deleted file mode 100644 index 7e8a03e8f..000000000 --- a/server/helpers/audit-logger.ts +++ /dev/null | |||
@@ -1,287 +0,0 @@ | |||
1 | import { diff } from 'deep-object-diff' | ||
2 | import express from 'express' | ||
3 | import flatten from 'flat' | ||
4 | import { chain } from 'lodash' | ||
5 | import { join } from 'path' | ||
6 | import { addColors, config, createLogger, format, transports } from 'winston' | ||
7 | import { AUDIT_LOG_FILENAME } from '@server/initializers/constants' | ||
8 | import { AdminAbuse, CustomConfig, User, VideoChannel, VideoChannelSync, VideoComment, VideoDetails, VideoImport } from '@shared/models' | ||
9 | import { CONFIG } from '../initializers/config' | ||
10 | import { jsonLoggerFormat, labelFormatter } from './logger' | ||
11 | |||
12 | function getAuditIdFromRes (res: express.Response) { | ||
13 | return res.locals.oauth.token.User.username | ||
14 | } | ||
15 | |||
16 | enum AUDIT_TYPE { | ||
17 | CREATE = 'create', | ||
18 | UPDATE = 'update', | ||
19 | DELETE = 'delete' | ||
20 | } | ||
21 | |||
22 | const colors = config.npm.colors | ||
23 | colors.audit = config.npm.colors.info | ||
24 | |||
25 | addColors(colors) | ||
26 | |||
27 | const auditLogger = createLogger({ | ||
28 | levels: { audit: 0 }, | ||
29 | transports: [ | ||
30 | new transports.File({ | ||
31 | filename: join(CONFIG.STORAGE.LOG_DIR, AUDIT_LOG_FILENAME), | ||
32 | level: 'audit', | ||
33 | maxsize: 5242880, | ||
34 | maxFiles: 5, | ||
35 | format: format.combine( | ||
36 | format.timestamp(), | ||
37 | labelFormatter(), | ||
38 | format.splat(), | ||
39 | jsonLoggerFormat | ||
40 | ) | ||
41 | }) | ||
42 | ], | ||
43 | exitOnError: true | ||
44 | }) | ||
45 | |||
46 | function auditLoggerWrapper (domain: string, user: string, action: AUDIT_TYPE, entity: EntityAuditView, oldEntity: EntityAuditView = null) { | ||
47 | let entityInfos: object | ||
48 | if (action === AUDIT_TYPE.UPDATE && oldEntity) { | ||
49 | const oldEntityKeys = oldEntity.toLogKeys() | ||
50 | const diffObject = diff(oldEntityKeys, entity.toLogKeys()) | ||
51 | const diffKeys = Object.entries(diffObject).reduce((newKeys, entry) => { | ||
52 | newKeys[`new-${entry[0]}`] = entry[1] | ||
53 | return newKeys | ||
54 | }, {}) | ||
55 | entityInfos = { ...oldEntityKeys, ...diffKeys } | ||
56 | } else { | ||
57 | entityInfos = { ...entity.toLogKeys() } | ||
58 | } | ||
59 | auditLogger.log('audit', JSON.stringify({ | ||
60 | user, | ||
61 | domain, | ||
62 | action, | ||
63 | ...entityInfos | ||
64 | })) | ||
65 | } | ||
66 | |||
67 | function auditLoggerFactory (domain: string) { | ||
68 | return { | ||
69 | create (user: string, entity: EntityAuditView) { | ||
70 | auditLoggerWrapper(domain, user, AUDIT_TYPE.CREATE, entity) | ||
71 | }, | ||
72 | update (user: string, entity: EntityAuditView, oldEntity: EntityAuditView) { | ||
73 | auditLoggerWrapper(domain, user, AUDIT_TYPE.UPDATE, entity, oldEntity) | ||
74 | }, | ||
75 | delete (user: string, entity: EntityAuditView) { | ||
76 | auditLoggerWrapper(domain, user, AUDIT_TYPE.DELETE, entity) | ||
77 | } | ||
78 | } | ||
79 | } | ||
80 | |||
81 | abstract class EntityAuditView { | ||
82 | constructor (private readonly keysToKeep: string[], private readonly prefix: string, private readonly entityInfos: object) { } | ||
83 | |||
84 | toLogKeys (): object { | ||
85 | return chain(flatten<object, any>(this.entityInfos, { delimiter: '-', safe: true })) | ||
86 | .pick(this.keysToKeep) | ||
87 | .mapKeys((_value, key) => `${this.prefix}-${key}`) | ||
88 | .value() | ||
89 | } | ||
90 | } | ||
91 | |||
92 | const videoKeysToKeep = [ | ||
93 | 'tags', | ||
94 | 'uuid', | ||
95 | 'id', | ||
96 | 'uuid', | ||
97 | 'createdAt', | ||
98 | 'updatedAt', | ||
99 | 'publishedAt', | ||
100 | 'category', | ||
101 | 'licence', | ||
102 | 'language', | ||
103 | 'privacy', | ||
104 | 'description', | ||
105 | 'duration', | ||
106 | 'isLocal', | ||
107 | 'name', | ||
108 | 'thumbnailPath', | ||
109 | 'previewPath', | ||
110 | 'nsfw', | ||
111 | 'waitTranscoding', | ||
112 | 'account-id', | ||
113 | 'account-uuid', | ||
114 | 'account-name', | ||
115 | 'channel-id', | ||
116 | 'channel-uuid', | ||
117 | 'channel-name', | ||
118 | 'support', | ||
119 | 'commentsEnabled', | ||
120 | 'downloadEnabled' | ||
121 | ] | ||
122 | class VideoAuditView extends EntityAuditView { | ||
123 | constructor (video: VideoDetails) { | ||
124 | super(videoKeysToKeep, 'video', video) | ||
125 | } | ||
126 | } | ||
127 | |||
128 | const videoImportKeysToKeep = [ | ||
129 | 'id', | ||
130 | 'targetUrl', | ||
131 | 'video-name' | ||
132 | ] | ||
133 | class VideoImportAuditView extends EntityAuditView { | ||
134 | constructor (videoImport: VideoImport) { | ||
135 | super(videoImportKeysToKeep, 'video-import', videoImport) | ||
136 | } | ||
137 | } | ||
138 | |||
139 | const commentKeysToKeep = [ | ||
140 | 'id', | ||
141 | 'text', | ||
142 | 'threadId', | ||
143 | 'inReplyToCommentId', | ||
144 | 'videoId', | ||
145 | 'createdAt', | ||
146 | 'updatedAt', | ||
147 | 'totalReplies', | ||
148 | 'account-id', | ||
149 | 'account-uuid', | ||
150 | 'account-name' | ||
151 | ] | ||
152 | class CommentAuditView extends EntityAuditView { | ||
153 | constructor (comment: VideoComment) { | ||
154 | super(commentKeysToKeep, 'comment', comment) | ||
155 | } | ||
156 | } | ||
157 | |||
158 | const userKeysToKeep = [ | ||
159 | 'id', | ||
160 | 'username', | ||
161 | 'email', | ||
162 | 'nsfwPolicy', | ||
163 | 'autoPlayVideo', | ||
164 | 'role', | ||
165 | 'videoQuota', | ||
166 | 'createdAt', | ||
167 | 'account-id', | ||
168 | 'account-uuid', | ||
169 | 'account-name', | ||
170 | 'account-followingCount', | ||
171 | 'account-followersCount', | ||
172 | 'account-createdAt', | ||
173 | 'account-updatedAt', | ||
174 | 'account-avatar-path', | ||
175 | 'account-avatar-createdAt', | ||
176 | 'account-avatar-updatedAt', | ||
177 | 'account-displayName', | ||
178 | 'account-description', | ||
179 | 'videoChannels' | ||
180 | ] | ||
181 | class UserAuditView extends EntityAuditView { | ||
182 | constructor (user: User) { | ||
183 | super(userKeysToKeep, 'user', user) | ||
184 | } | ||
185 | } | ||
186 | |||
187 | const channelKeysToKeep = [ | ||
188 | 'id', | ||
189 | 'uuid', | ||
190 | 'name', | ||
191 | 'followingCount', | ||
192 | 'followersCount', | ||
193 | 'createdAt', | ||
194 | 'updatedAt', | ||
195 | 'avatar-path', | ||
196 | 'avatar-createdAt', | ||
197 | 'avatar-updatedAt', | ||
198 | 'displayName', | ||
199 | 'description', | ||
200 | 'support', | ||
201 | 'isLocal', | ||
202 | 'ownerAccount-id', | ||
203 | 'ownerAccount-uuid', | ||
204 | 'ownerAccount-name', | ||
205 | 'ownerAccount-displayedName' | ||
206 | ] | ||
207 | class VideoChannelAuditView extends EntityAuditView { | ||
208 | constructor (channel: VideoChannel) { | ||
209 | super(channelKeysToKeep, 'channel', channel) | ||
210 | } | ||
211 | } | ||
212 | |||
213 | const abuseKeysToKeep = [ | ||
214 | 'id', | ||
215 | 'reason', | ||
216 | 'reporterAccount', | ||
217 | 'createdAt' | ||
218 | ] | ||
219 | class AbuseAuditView extends EntityAuditView { | ||
220 | constructor (abuse: AdminAbuse) { | ||
221 | super(abuseKeysToKeep, 'abuse', abuse) | ||
222 | } | ||
223 | } | ||
224 | |||
225 | const customConfigKeysToKeep = [ | ||
226 | 'instance-name', | ||
227 | 'instance-shortDescription', | ||
228 | 'instance-description', | ||
229 | 'instance-terms', | ||
230 | 'instance-defaultClientRoute', | ||
231 | 'instance-defaultNSFWPolicy', | ||
232 | 'instance-customizations-javascript', | ||
233 | 'instance-customizations-css', | ||
234 | 'services-twitter-username', | ||
235 | 'services-twitter-whitelisted', | ||
236 | 'cache-previews-size', | ||
237 | 'cache-captions-size', | ||
238 | 'signup-enabled', | ||
239 | 'signup-limit', | ||
240 | 'signup-requiresEmailVerification', | ||
241 | 'admin-email', | ||
242 | 'user-videoQuota', | ||
243 | 'transcoding-enabled', | ||
244 | 'transcoding-threads', | ||
245 | 'transcoding-resolutions' | ||
246 | ] | ||
247 | class CustomConfigAuditView extends EntityAuditView { | ||
248 | constructor (customConfig: CustomConfig) { | ||
249 | const infos: any = customConfig | ||
250 | const resolutionsDict = infos.transcoding.resolutions | ||
251 | const resolutionsArray = [] | ||
252 | |||
253 | Object.entries(resolutionsDict) | ||
254 | .forEach(([ resolution, isEnabled ]) => { | ||
255 | if (isEnabled) resolutionsArray.push(resolution) | ||
256 | }) | ||
257 | |||
258 | Object.assign({}, infos, { transcoding: { resolutions: resolutionsArray } }) | ||
259 | super(customConfigKeysToKeep, 'config', infos) | ||
260 | } | ||
261 | } | ||
262 | |||
263 | const channelSyncKeysToKeep = [ | ||
264 | 'id', | ||
265 | 'externalChannelUrl', | ||
266 | 'channel-id', | ||
267 | 'channel-name' | ||
268 | ] | ||
269 | class VideoChannelSyncAuditView extends EntityAuditView { | ||
270 | constructor (channelSync: VideoChannelSync) { | ||
271 | super(channelSyncKeysToKeep, 'channelSync', channelSync) | ||
272 | } | ||
273 | } | ||
274 | |||
275 | export { | ||
276 | getAuditIdFromRes, | ||
277 | |||
278 | auditLoggerFactory, | ||
279 | VideoImportAuditView, | ||
280 | VideoChannelAuditView, | ||
281 | CommentAuditView, | ||
282 | UserAuditView, | ||
283 | VideoAuditView, | ||
284 | AbuseAuditView, | ||
285 | CustomConfigAuditView, | ||
286 | VideoChannelSyncAuditView | ||
287 | } | ||
diff --git a/server/helpers/captions-utils.ts b/server/helpers/captions-utils.ts deleted file mode 100644 index f6e5b9784..000000000 --- a/server/helpers/captions-utils.ts +++ /dev/null | |||
@@ -1,53 +0,0 @@ | |||
1 | import { createReadStream, createWriteStream, move, remove } from 'fs-extra' | ||
2 | import { join } from 'path' | ||
3 | import srt2vtt from 'srt-to-vtt' | ||
4 | import { Transform } from 'stream' | ||
5 | import { MVideoCaption } from '@server/types/models' | ||
6 | import { CONFIG } from '../initializers/config' | ||
7 | import { pipelinePromise } from './core-utils' | ||
8 | |||
9 | async function moveAndProcessCaptionFile (physicalFile: { filename: string, path: string }, videoCaption: MVideoCaption) { | ||
10 | const videoCaptionsDir = CONFIG.STORAGE.CAPTIONS_DIR | ||
11 | const destination = join(videoCaptionsDir, videoCaption.filename) | ||
12 | |||
13 | // Convert this srt file to vtt | ||
14 | if (physicalFile.path.endsWith('.srt')) { | ||
15 | await convertSrtToVtt(physicalFile.path, destination) | ||
16 | await remove(physicalFile.path) | ||
17 | } else if (physicalFile.path !== destination) { // Just move the vtt file | ||
18 | await move(physicalFile.path, destination, { overwrite: true }) | ||
19 | } | ||
20 | |||
21 | // This is important in case if there is another attempt in the retry process | ||
22 | physicalFile.filename = videoCaption.filename | ||
23 | physicalFile.path = destination | ||
24 | } | ||
25 | |||
26 | // --------------------------------------------------------------------------- | ||
27 | |||
28 | export { | ||
29 | moveAndProcessCaptionFile | ||
30 | } | ||
31 | |||
32 | // --------------------------------------------------------------------------- | ||
33 | |||
34 | function convertSrtToVtt (source: string, destination: string) { | ||
35 | const fixVTT = new Transform({ | ||
36 | transform: (chunk, _encoding, cb) => { | ||
37 | let block: string = chunk.toString() | ||
38 | |||
39 | block = block.replace(/(\d\d:\d\d:\d\d)(\s)/g, '$1.000$2') | ||
40 | .replace(/(\d\d:\d\d:\d\d),(\d)(\s)/g, '$1.00$2$3') | ||
41 | .replace(/(\d\d:\d\d:\d\d),(\d\d)(\s)/g, '$1.0$2$3') | ||
42 | |||
43 | return cb(undefined, block) | ||
44 | } | ||
45 | }) | ||
46 | |||
47 | return pipelinePromise( | ||
48 | createReadStream(source), | ||
49 | srt2vtt(), | ||
50 | fixVTT, | ||
51 | createWriteStream(destination) | ||
52 | ) | ||
53 | } | ||
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts deleted file mode 100644 index 242c49e89..000000000 --- a/server/helpers/core-utils.ts +++ /dev/null | |||
@@ -1,315 +0,0 @@ | |||
1 | /* eslint-disable no-useless-call */ | ||
2 | |||
3 | /* | ||
4 | Different from 'utils' because we don't import other PeerTube modules. | ||
5 | Useful to avoid circular dependencies. | ||
6 | */ | ||
7 | |||
8 | import { exec, ExecOptions } from 'child_process' | ||
9 | import { ED25519KeyPairOptions, generateKeyPair, randomBytes, RSAKeyPairOptions, scrypt } from 'crypto' | ||
10 | import { truncate } from 'lodash' | ||
11 | import { pipeline } from 'stream' | ||
12 | import { URL } from 'url' | ||
13 | import { promisify } from 'util' | ||
14 | import { promisify1, promisify2, promisify3 } from '@shared/core-utils' | ||
15 | |||
16 | const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => { | ||
17 | if (!oldObject || typeof oldObject !== 'object') { | ||
18 | return valueConverter(oldObject) | ||
19 | } | ||
20 | |||
21 | if (Array.isArray(oldObject)) { | ||
22 | return oldObject.map(e => objectConverter(e, keyConverter, valueConverter)) | ||
23 | } | ||
24 | |||
25 | const newObject = {} | ||
26 | Object.keys(oldObject).forEach(oldKey => { | ||
27 | const newKey = keyConverter(oldKey) | ||
28 | newObject[newKey] = objectConverter(oldObject[oldKey], keyConverter, valueConverter) | ||
29 | }) | ||
30 | |||
31 | return newObject | ||
32 | } | ||
33 | |||
34 | function mapToJSON (map: Map<any, any>) { | ||
35 | const obj: any = {} | ||
36 | |||
37 | for (const [ k, v ] of map) { | ||
38 | obj[k] = v | ||
39 | } | ||
40 | |||
41 | return obj | ||
42 | } | ||
43 | |||
44 | // --------------------------------------------------------------------------- | ||
45 | |||
46 | const timeTable = { | ||
47 | ms: 1, | ||
48 | second: 1000, | ||
49 | minute: 60000, | ||
50 | hour: 3600000, | ||
51 | day: 3600000 * 24, | ||
52 | week: 3600000 * 24 * 7, | ||
53 | month: 3600000 * 24 * 30 | ||
54 | } | ||
55 | |||
56 | export function parseDurationToMs (duration: number | string): number { | ||
57 | if (duration === null) return null | ||
58 | if (typeof duration === 'number') return duration | ||
59 | if (!isNaN(+duration)) return +duration | ||
60 | |||
61 | if (typeof duration === 'string') { | ||
62 | const split = duration.match(/^([\d.,]+)\s?(\w+)$/) | ||
63 | |||
64 | if (split.length === 3) { | ||
65 | const len = parseFloat(split[1]) | ||
66 | let unit = split[2].replace(/s$/i, '').toLowerCase() | ||
67 | if (unit === 'm') { | ||
68 | unit = 'ms' | ||
69 | } | ||
70 | |||
71 | return (len || 1) * (timeTable[unit] || 0) | ||
72 | } | ||
73 | } | ||
74 | |||
75 | throw new Error(`Duration ${duration} could not be properly parsed`) | ||
76 | } | ||
77 | |||
78 | export function parseBytes (value: string | number): number { | ||
79 | if (typeof value === 'number') return value | ||
80 | if (!isNaN(+value)) return +value | ||
81 | |||
82 | const tgm = /^(\d+)\s*TB\s*(\d+)\s*GB\s*(\d+)\s*MB$/ | ||
83 | const tg = /^(\d+)\s*TB\s*(\d+)\s*GB$/ | ||
84 | const tm = /^(\d+)\s*TB\s*(\d+)\s*MB$/ | ||
85 | const gm = /^(\d+)\s*GB\s*(\d+)\s*MB$/ | ||
86 | const t = /^(\d+)\s*TB$/ | ||
87 | const g = /^(\d+)\s*GB$/ | ||
88 | const m = /^(\d+)\s*MB$/ | ||
89 | const b = /^(\d+)\s*B$/ | ||
90 | |||
91 | let match: RegExpMatchArray | ||
92 | |||
93 | if (value.match(tgm)) { | ||
94 | match = value.match(tgm) | ||
95 | return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 + | ||
96 | parseInt(match[2], 10) * 1024 * 1024 * 1024 + | ||
97 | parseInt(match[3], 10) * 1024 * 1024 | ||
98 | } | ||
99 | |||
100 | if (value.match(tg)) { | ||
101 | match = value.match(tg) | ||
102 | return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 + | ||
103 | parseInt(match[2], 10) * 1024 * 1024 * 1024 | ||
104 | } | ||
105 | |||
106 | if (value.match(tm)) { | ||
107 | match = value.match(tm) | ||
108 | return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 + | ||
109 | parseInt(match[2], 10) * 1024 * 1024 | ||
110 | } | ||
111 | |||
112 | if (value.match(gm)) { | ||
113 | match = value.match(gm) | ||
114 | return parseInt(match[1], 10) * 1024 * 1024 * 1024 + | ||
115 | parseInt(match[2], 10) * 1024 * 1024 | ||
116 | } | ||
117 | |||
118 | if (value.match(t)) { | ||
119 | match = value.match(t) | ||
120 | return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 | ||
121 | } | ||
122 | |||
123 | if (value.match(g)) { | ||
124 | match = value.match(g) | ||
125 | return parseInt(match[1], 10) * 1024 * 1024 * 1024 | ||
126 | } | ||
127 | |||
128 | if (value.match(m)) { | ||
129 | match = value.match(m) | ||
130 | return parseInt(match[1], 10) * 1024 * 1024 | ||
131 | } | ||
132 | |||
133 | if (value.match(b)) { | ||
134 | match = value.match(b) | ||
135 | return parseInt(match[1], 10) * 1024 | ||
136 | } | ||
137 | |||
138 | return parseInt(value, 10) | ||
139 | } | ||
140 | |||
141 | // --------------------------------------------------------------------------- | ||
142 | |||
143 | function sanitizeUrl (url: string) { | ||
144 | const urlObject = new URL(url) | ||
145 | |||
146 | if (urlObject.protocol === 'https:' && urlObject.port === '443') { | ||
147 | urlObject.port = '' | ||
148 | } else if (urlObject.protocol === 'http:' && urlObject.port === '80') { | ||
149 | urlObject.port = '' | ||
150 | } | ||
151 | |||
152 | return urlObject.href.replace(/\/$/, '') | ||
153 | } | ||
154 | |||
155 | // Don't import remote scheme from constants because we are in core utils | ||
156 | function sanitizeHost (host: string, remoteScheme: string) { | ||
157 | const toRemove = remoteScheme === 'https' ? 443 : 80 | ||
158 | |||
159 | return host.replace(new RegExp(`:${toRemove}$`), '') | ||
160 | } | ||
161 | |||
162 | // --------------------------------------------------------------------------- | ||
163 | |||
164 | function isTestInstance () { | ||
165 | return process.env.NODE_ENV === 'test' | ||
166 | } | ||
167 | |||
168 | function isDevInstance () { | ||
169 | return process.env.NODE_ENV === 'dev' | ||
170 | } | ||
171 | |||
172 | function isTestOrDevInstance () { | ||
173 | return isTestInstance() || isDevInstance() | ||
174 | } | ||
175 | |||
176 | function isProdInstance () { | ||
177 | return process.env.NODE_ENV === 'production' | ||
178 | } | ||
179 | |||
180 | function getAppNumber () { | ||
181 | return process.env.NODE_APP_INSTANCE || '' | ||
182 | } | ||
183 | |||
184 | // --------------------------------------------------------------------------- | ||
185 | |||
186 | // Consistent with .length, lodash truncate function is not | ||
187 | function peertubeTruncate (str: string, options: { length: number, separator?: RegExp, omission?: string }) { | ||
188 | const truncatedStr = truncate(str, options) | ||
189 | |||
190 | // The truncated string is okay, we can return it | ||
191 | if (truncatedStr.length <= options.length) return truncatedStr | ||
192 | |||
193 | // Lodash takes into account all UTF characters, whereas String.prototype.length does not: some characters have a length of 2 | ||
194 | // We always use the .length so we need to truncate more if needed | ||
195 | options.length -= truncatedStr.length - options.length | ||
196 | return truncate(str, options) | ||
197 | } | ||
198 | |||
199 | function pageToStartAndCount (page: number, itemsPerPage: number) { | ||
200 | const start = (page - 1) * itemsPerPage | ||
201 | |||
202 | return { start, count: itemsPerPage } | ||
203 | } | ||
204 | |||
205 | // --------------------------------------------------------------------------- | ||
206 | |||
207 | type SemVersion = { major: number, minor: number, patch: number } | ||
208 | function parseSemVersion (s: string) { | ||
209 | const parsed = s.match(/^v?(\d+)\.(\d+)\.(\d+)$/i) | ||
210 | |||
211 | return { | ||
212 | major: parseInt(parsed[1]), | ||
213 | minor: parseInt(parsed[2]), | ||
214 | patch: parseInt(parsed[3]) | ||
215 | } as SemVersion | ||
216 | } | ||
217 | |||
218 | // --------------------------------------------------------------------------- | ||
219 | |||
220 | function execShell (command: string, options?: ExecOptions) { | ||
221 | return new Promise<{ err?: Error, stdout: string, stderr: string }>((res, rej) => { | ||
222 | exec(command, options, (err, stdout, stderr) => { | ||
223 | // eslint-disable-next-line prefer-promise-reject-errors | ||
224 | if (err) return rej({ err, stdout, stderr }) | ||
225 | |||
226 | return res({ stdout, stderr }) | ||
227 | }) | ||
228 | }) | ||
229 | } | ||
230 | |||
231 | // --------------------------------------------------------------------------- | ||
232 | |||
233 | function generateRSAKeyPairPromise (size: number) { | ||
234 | return new Promise<{ publicKey: string, privateKey: string }>((res, rej) => { | ||
235 | const options: RSAKeyPairOptions<'pem', 'pem'> = { | ||
236 | modulusLength: size, | ||
237 | publicKeyEncoding: { | ||
238 | type: 'spki', | ||
239 | format: 'pem' | ||
240 | }, | ||
241 | privateKeyEncoding: { | ||
242 | type: 'pkcs1', | ||
243 | format: 'pem' | ||
244 | } | ||
245 | } | ||
246 | |||
247 | generateKeyPair('rsa', options, (err, publicKey, privateKey) => { | ||
248 | if (err) return rej(err) | ||
249 | |||
250 | return res({ publicKey, privateKey }) | ||
251 | }) | ||
252 | }) | ||
253 | } | ||
254 | |||
255 | function generateED25519KeyPairPromise () { | ||
256 | return new Promise<{ publicKey: string, privateKey: string }>((res, rej) => { | ||
257 | const options: ED25519KeyPairOptions<'pem', 'pem'> = { | ||
258 | publicKeyEncoding: { | ||
259 | type: 'spki', | ||
260 | format: 'pem' | ||
261 | }, | ||
262 | privateKeyEncoding: { | ||
263 | type: 'pkcs8', | ||
264 | format: 'pem' | ||
265 | } | ||
266 | } | ||
267 | |||
268 | generateKeyPair('ed25519', options, (err, publicKey, privateKey) => { | ||
269 | if (err) return rej(err) | ||
270 | |||
271 | return res({ publicKey, privateKey }) | ||
272 | }) | ||
273 | }) | ||
274 | } | ||
275 | |||
276 | // --------------------------------------------------------------------------- | ||
277 | |||
278 | const randomBytesPromise = promisify1<number, Buffer>(randomBytes) | ||
279 | const scryptPromise = promisify3<string, string, number, Buffer>(scrypt) | ||
280 | const execPromise2 = promisify2<string, any, string>(exec) | ||
281 | const execPromise = promisify1<string, string>(exec) | ||
282 | const pipelinePromise = promisify(pipeline) | ||
283 | |||
284 | // --------------------------------------------------------------------------- | ||
285 | |||
286 | export { | ||
287 | isTestInstance, | ||
288 | isTestOrDevInstance, | ||
289 | isProdInstance, | ||
290 | getAppNumber, | ||
291 | |||
292 | objectConverter, | ||
293 | mapToJSON, | ||
294 | |||
295 | sanitizeUrl, | ||
296 | sanitizeHost, | ||
297 | |||
298 | execShell, | ||
299 | |||
300 | pageToStartAndCount, | ||
301 | peertubeTruncate, | ||
302 | |||
303 | scryptPromise, | ||
304 | |||
305 | randomBytesPromise, | ||
306 | |||
307 | generateRSAKeyPairPromise, | ||
308 | generateED25519KeyPairPromise, | ||
309 | |||
310 | execPromise2, | ||
311 | execPromise, | ||
312 | pipelinePromise, | ||
313 | |||
314 | parseSemVersion | ||
315 | } | ||
diff --git a/server/helpers/custom-jsonld-signature.ts b/server/helpers/custom-jsonld-signature.ts deleted file mode 100644 index 3c706e372..000000000 --- a/server/helpers/custom-jsonld-signature.ts +++ /dev/null | |||
@@ -1,91 +0,0 @@ | |||
1 | import AsyncLRU from 'async-lru' | ||
2 | import { logger } from './logger' | ||
3 | |||
4 | import jsonld = require('jsonld') | ||
5 | |||
6 | const CACHE = { | ||
7 | 'https://w3id.org/security/v1': { | ||
8 | '@context': { | ||
9 | id: '@id', | ||
10 | type: '@type', | ||
11 | |||
12 | dc: 'http://purl.org/dc/terms/', | ||
13 | sec: 'https://w3id.org/security#', | ||
14 | xsd: 'http://www.w3.org/2001/XMLSchema#', | ||
15 | |||
16 | EcdsaKoblitzSignature2016: 'sec:EcdsaKoblitzSignature2016', | ||
17 | Ed25519Signature2018: 'sec:Ed25519Signature2018', | ||
18 | EncryptedMessage: 'sec:EncryptedMessage', | ||
19 | GraphSignature2012: 'sec:GraphSignature2012', | ||
20 | LinkedDataSignature2015: 'sec:LinkedDataSignature2015', | ||
21 | LinkedDataSignature2016: 'sec:LinkedDataSignature2016', | ||
22 | CryptographicKey: 'sec:Key', | ||
23 | |||
24 | authenticationTag: 'sec:authenticationTag', | ||
25 | canonicalizationAlgorithm: 'sec:canonicalizationAlgorithm', | ||
26 | cipherAlgorithm: 'sec:cipherAlgorithm', | ||
27 | cipherData: 'sec:cipherData', | ||
28 | cipherKey: 'sec:cipherKey', | ||
29 | created: { '@id': 'dc:created', '@type': 'xsd:dateTime' }, | ||
30 | creator: { '@id': 'dc:creator', '@type': '@id' }, | ||
31 | digestAlgorithm: 'sec:digestAlgorithm', | ||
32 | digestValue: 'sec:digestValue', | ||
33 | domain: 'sec:domain', | ||
34 | encryptionKey: 'sec:encryptionKey', | ||
35 | expiration: { '@id': 'sec:expiration', '@type': 'xsd:dateTime' }, | ||
36 | expires: { '@id': 'sec:expiration', '@type': 'xsd:dateTime' }, | ||
37 | initializationVector: 'sec:initializationVector', | ||
38 | iterationCount: 'sec:iterationCount', | ||
39 | nonce: 'sec:nonce', | ||
40 | normalizationAlgorithm: 'sec:normalizationAlgorithm', | ||
41 | owner: { '@id': 'sec:owner', '@type': '@id' }, | ||
42 | password: 'sec:password', | ||
43 | privateKey: { '@id': 'sec:privateKey', '@type': '@id' }, | ||
44 | privateKeyPem: 'sec:privateKeyPem', | ||
45 | publicKey: { '@id': 'sec:publicKey', '@type': '@id' }, | ||
46 | publicKeyBase58: 'sec:publicKeyBase58', | ||
47 | publicKeyPem: 'sec:publicKeyPem', | ||
48 | publicKeyWif: 'sec:publicKeyWif', | ||
49 | publicKeyService: { '@id': 'sec:publicKeyService', '@type': '@id' }, | ||
50 | revoked: { '@id': 'sec:revoked', '@type': 'xsd:dateTime' }, | ||
51 | salt: 'sec:salt', | ||
52 | signature: 'sec:signature', | ||
53 | signatureAlgorithm: 'sec:signingAlgorithm', | ||
54 | signatureValue: 'sec:signatureValue' | ||
55 | } | ||
56 | } | ||
57 | } | ||
58 | |||
59 | const nodeDocumentLoader = jsonld.documentLoaders.node() | ||
60 | |||
61 | const lru = new AsyncLRU({ | ||
62 | max: 10, | ||
63 | load: (url, cb) => { | ||
64 | if (CACHE[url] !== undefined) { | ||
65 | logger.debug('Using cache for JSON-LD %s.', url) | ||
66 | |||
67 | return cb(null, { | ||
68 | contextUrl: null, | ||
69 | document: CACHE[url], | ||
70 | documentUrl: url | ||
71 | }) | ||
72 | } | ||
73 | |||
74 | nodeDocumentLoader(url) | ||
75 | .then(value => cb(null, value)) | ||
76 | .catch(err => cb(err)) | ||
77 | } | ||
78 | }) | ||
79 | |||
80 | /* eslint-disable no-import-assign */ | ||
81 | jsonld.documentLoader = (url) => { | ||
82 | return new Promise((res, rej) => { | ||
83 | lru.get(url, (err, value) => { | ||
84 | if (err) return rej(err) | ||
85 | |||
86 | return res(value) | ||
87 | }) | ||
88 | }) | ||
89 | } | ||
90 | |||
91 | export { jsonld } | ||
diff --git a/server/helpers/custom-validators/abuses.ts b/server/helpers/custom-validators/abuses.ts deleted file mode 100644 index 94719641a..000000000 --- a/server/helpers/custom-validators/abuses.ts +++ /dev/null | |||
@@ -1,68 +0,0 @@ | |||
1 | import validator from 'validator' | ||
2 | import { abusePredefinedReasonsMap } from '@shared/core-utils/abuse' | ||
3 | import { AbuseCreate, AbuseFilter, AbusePredefinedReasonsString, AbuseVideoIs } from '@shared/models' | ||
4 | import { ABUSE_STATES, CONSTRAINTS_FIELDS } from '../../initializers/constants' | ||
5 | import { exists, isArray } from './misc' | ||
6 | |||
7 | const ABUSES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.ABUSES | ||
8 | const ABUSE_MESSAGES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.ABUSE_MESSAGES | ||
9 | |||
10 | function isAbuseReasonValid (value: string) { | ||
11 | return exists(value) && validator.isLength(value, ABUSES_CONSTRAINTS_FIELDS.REASON) | ||
12 | } | ||
13 | |||
14 | function isAbusePredefinedReasonValid (value: AbusePredefinedReasonsString) { | ||
15 | return exists(value) && value in abusePredefinedReasonsMap | ||
16 | } | ||
17 | |||
18 | function isAbuseFilterValid (value: AbuseFilter) { | ||
19 | return value === 'video' || value === 'comment' || value === 'account' | ||
20 | } | ||
21 | |||
22 | function areAbusePredefinedReasonsValid (value: AbusePredefinedReasonsString[]) { | ||
23 | return exists(value) && isArray(value) && value.every(v => v in abusePredefinedReasonsMap) | ||
24 | } | ||
25 | |||
26 | function isAbuseTimestampValid (value: number) { | ||
27 | return value === null || (exists(value) && validator.isInt('' + value, { min: 0 })) | ||
28 | } | ||
29 | |||
30 | function isAbuseTimestampCoherent (endAt: number, { req }) { | ||
31 | const startAt = (req.body as AbuseCreate).video.startAt | ||
32 | |||
33 | return exists(startAt) && endAt > startAt | ||
34 | } | ||
35 | |||
36 | function isAbuseModerationCommentValid (value: string) { | ||
37 | return exists(value) && validator.isLength(value, ABUSES_CONSTRAINTS_FIELDS.MODERATION_COMMENT) | ||
38 | } | ||
39 | |||
40 | function isAbuseStateValid (value: string) { | ||
41 | return exists(value) && ABUSE_STATES[value] !== undefined | ||
42 | } | ||
43 | |||
44 | function isAbuseVideoIsValid (value: AbuseVideoIs) { | ||
45 | return exists(value) && ( | ||
46 | value === 'deleted' || | ||
47 | value === 'blacklisted' | ||
48 | ) | ||
49 | } | ||
50 | |||
51 | function isAbuseMessageValid (value: string) { | ||
52 | return exists(value) && validator.isLength(value, ABUSE_MESSAGES_CONSTRAINTS_FIELDS.MESSAGE) | ||
53 | } | ||
54 | |||
55 | // --------------------------------------------------------------------------- | ||
56 | |||
57 | export { | ||
58 | isAbuseReasonValid, | ||
59 | isAbuseFilterValid, | ||
60 | isAbusePredefinedReasonValid, | ||
61 | isAbuseMessageValid, | ||
62 | areAbusePredefinedReasonsValid, | ||
63 | isAbuseTimestampValid, | ||
64 | isAbuseTimestampCoherent, | ||
65 | isAbuseModerationCommentValid, | ||
66 | isAbuseStateValid, | ||
67 | isAbuseVideoIsValid | ||
68 | } | ||
diff --git a/server/helpers/custom-validators/accounts.ts b/server/helpers/custom-validators/accounts.ts deleted file mode 100644 index f676669ea..000000000 --- a/server/helpers/custom-validators/accounts.ts +++ /dev/null | |||
@@ -1,22 +0,0 @@ | |||
1 | import { isUserDescriptionValid, isUserUsernameValid } from './users' | ||
2 | import { exists } from './misc' | ||
3 | |||
4 | function isAccountNameValid (value: string) { | ||
5 | return isUserUsernameValid(value) | ||
6 | } | ||
7 | |||
8 | function isAccountIdValid (value: string) { | ||
9 | return exists(value) | ||
10 | } | ||
11 | |||
12 | function isAccountDescriptionValid (value: string) { | ||
13 | return isUserDescriptionValid(value) | ||
14 | } | ||
15 | |||
16 | // --------------------------------------------------------------------------- | ||
17 | |||
18 | export { | ||
19 | isAccountIdValid, | ||
20 | isAccountDescriptionValid, | ||
21 | isAccountNameValid | ||
22 | } | ||
diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts deleted file mode 100644 index 90a918523..000000000 --- a/server/helpers/custom-validators/activitypub/activity.ts +++ /dev/null | |||
@@ -1,151 +0,0 @@ | |||
1 | import validator from 'validator' | ||
2 | import { Activity, ActivityType } from '../../../../shared/models/activitypub' | ||
3 | import { isAbuseReasonValid } from '../abuses' | ||
4 | import { exists } from '../misc' | ||
5 | import { sanitizeAndCheckActorObject } from './actor' | ||
6 | import { isCacheFileObjectValid } from './cache-file' | ||
7 | import { isActivityPubUrlValid, isBaseActivityValid, isObjectValid } from './misc' | ||
8 | import { isPlaylistObjectValid } from './playlist' | ||
9 | import { sanitizeAndCheckVideoCommentObject } from './video-comments' | ||
10 | import { sanitizeAndCheckVideoTorrentObject } from './videos' | ||
11 | import { isWatchActionObjectValid } from './watch-action' | ||
12 | |||
13 | function isRootActivityValid (activity: any) { | ||
14 | return isCollection(activity) || isActivity(activity) | ||
15 | } | ||
16 | |||
17 | function isCollection (activity: any) { | ||
18 | return (activity.type === 'Collection' || activity.type === 'OrderedCollection') && | ||
19 | validator.isInt(activity.totalItems, { min: 0 }) && | ||
20 | Array.isArray(activity.items) | ||
21 | } | ||
22 | |||
23 | function isActivity (activity: any) { | ||
24 | return isActivityPubUrlValid(activity.id) && | ||
25 | exists(activity.actor) && | ||
26 | (isActivityPubUrlValid(activity.actor) || isActivityPubUrlValid(activity.actor.id)) | ||
27 | } | ||
28 | |||
29 | const activityCheckers: { [ P in ActivityType ]: (activity: Activity) => boolean } = { | ||
30 | Create: isCreateActivityValid, | ||
31 | Update: isUpdateActivityValid, | ||
32 | Delete: isDeleteActivityValid, | ||
33 | Follow: isFollowActivityValid, | ||
34 | Accept: isAcceptActivityValid, | ||
35 | Reject: isRejectActivityValid, | ||
36 | Announce: isAnnounceActivityValid, | ||
37 | Undo: isUndoActivityValid, | ||
38 | Like: isLikeActivityValid, | ||
39 | View: isViewActivityValid, | ||
40 | Flag: isFlagActivityValid, | ||
41 | Dislike: isDislikeActivityValid | ||
42 | } | ||
43 | |||
44 | function isActivityValid (activity: any) { | ||
45 | const checker = activityCheckers[activity.type] | ||
46 | // Unknown activity type | ||
47 | if (!checker) return false | ||
48 | |||
49 | return checker(activity) | ||
50 | } | ||
51 | |||
52 | function isFlagActivityValid (activity: any) { | ||
53 | return isBaseActivityValid(activity, 'Flag') && | ||
54 | isAbuseReasonValid(activity.content) && | ||
55 | isActivityPubUrlValid(activity.object) | ||
56 | } | ||
57 | |||
58 | function isLikeActivityValid (activity: any) { | ||
59 | return isBaseActivityValid(activity, 'Like') && | ||
60 | isObjectValid(activity.object) | ||
61 | } | ||
62 | |||
63 | function isDislikeActivityValid (activity: any) { | ||
64 | return isBaseActivityValid(activity, 'Dislike') && | ||
65 | isObjectValid(activity.object) | ||
66 | } | ||
67 | |||
68 | function isAnnounceActivityValid (activity: any) { | ||
69 | return isBaseActivityValid(activity, 'Announce') && | ||
70 | isObjectValid(activity.object) | ||
71 | } | ||
72 | |||
73 | function isViewActivityValid (activity: any) { | ||
74 | return isBaseActivityValid(activity, 'View') && | ||
75 | isActivityPubUrlValid(activity.actor) && | ||
76 | isActivityPubUrlValid(activity.object) | ||
77 | } | ||
78 | |||
79 | function isCreateActivityValid (activity: any) { | ||
80 | return isBaseActivityValid(activity, 'Create') && | ||
81 | ( | ||
82 | isViewActivityValid(activity.object) || | ||
83 | isDislikeActivityValid(activity.object) || | ||
84 | isFlagActivityValid(activity.object) || | ||
85 | isPlaylistObjectValid(activity.object) || | ||
86 | isWatchActionObjectValid(activity.object) || | ||
87 | |||
88 | isCacheFileObjectValid(activity.object) || | ||
89 | sanitizeAndCheckVideoCommentObject(activity.object) || | ||
90 | sanitizeAndCheckVideoTorrentObject(activity.object) | ||
91 | ) | ||
92 | } | ||
93 | |||
94 | function isUpdateActivityValid (activity: any) { | ||
95 | return isBaseActivityValid(activity, 'Update') && | ||
96 | ( | ||
97 | isCacheFileObjectValid(activity.object) || | ||
98 | isPlaylistObjectValid(activity.object) || | ||
99 | sanitizeAndCheckVideoTorrentObject(activity.object) || | ||
100 | sanitizeAndCheckActorObject(activity.object) | ||
101 | ) | ||
102 | } | ||
103 | |||
104 | function isDeleteActivityValid (activity: any) { | ||
105 | // We don't really check objects | ||
106 | return isBaseActivityValid(activity, 'Delete') && | ||
107 | isObjectValid(activity.object) | ||
108 | } | ||
109 | |||
110 | function isFollowActivityValid (activity: any) { | ||
111 | return isBaseActivityValid(activity, 'Follow') && | ||
112 | isObjectValid(activity.object) | ||
113 | } | ||
114 | |||
115 | function isAcceptActivityValid (activity: any) { | ||
116 | return isBaseActivityValid(activity, 'Accept') | ||
117 | } | ||
118 | |||
119 | function isRejectActivityValid (activity: any) { | ||
120 | return isBaseActivityValid(activity, 'Reject') | ||
121 | } | ||
122 | |||
123 | function isUndoActivityValid (activity: any) { | ||
124 | return isBaseActivityValid(activity, 'Undo') && | ||
125 | ( | ||
126 | isFollowActivityValid(activity.object) || | ||
127 | isLikeActivityValid(activity.object) || | ||
128 | isDislikeActivityValid(activity.object) || | ||
129 | isAnnounceActivityValid(activity.object) || | ||
130 | isCreateActivityValid(activity.object) | ||
131 | ) | ||
132 | } | ||
133 | |||
134 | // --------------------------------------------------------------------------- | ||
135 | |||
136 | export { | ||
137 | isRootActivityValid, | ||
138 | isActivityValid, | ||
139 | isFlagActivityValid, | ||
140 | isLikeActivityValid, | ||
141 | isDislikeActivityValid, | ||
142 | isAnnounceActivityValid, | ||
143 | isViewActivityValid, | ||
144 | isCreateActivityValid, | ||
145 | isUpdateActivityValid, | ||
146 | isDeleteActivityValid, | ||
147 | isFollowActivityValid, | ||
148 | isAcceptActivityValid, | ||
149 | isRejectActivityValid, | ||
150 | isUndoActivityValid | ||
151 | } | ||
diff --git a/server/helpers/custom-validators/activitypub/actor.ts b/server/helpers/custom-validators/activitypub/actor.ts deleted file mode 100644 index f43c35b23..000000000 --- a/server/helpers/custom-validators/activitypub/actor.ts +++ /dev/null | |||
@@ -1,142 +0,0 @@ | |||
1 | import validator from 'validator' | ||
2 | import { CONSTRAINTS_FIELDS } from '../../../initializers/constants' | ||
3 | import { exists, isArray, isDateValid } from '../misc' | ||
4 | import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc' | ||
5 | import { isHostValid } from '../servers' | ||
6 | import { peertubeTruncate } from '@server/helpers/core-utils' | ||
7 | |||
8 | function isActorEndpointsObjectValid (endpointObject: any) { | ||
9 | if (endpointObject?.sharedInbox) { | ||
10 | return isActivityPubUrlValid(endpointObject.sharedInbox) | ||
11 | } | ||
12 | |||
13 | // Shared inbox is optional | ||
14 | return true | ||
15 | } | ||
16 | |||
17 | function isActorPublicKeyObjectValid (publicKeyObject: any) { | ||
18 | return isActivityPubUrlValid(publicKeyObject.id) && | ||
19 | isActivityPubUrlValid(publicKeyObject.owner) && | ||
20 | isActorPublicKeyValid(publicKeyObject.publicKeyPem) | ||
21 | } | ||
22 | |||
23 | function isActorTypeValid (type: string) { | ||
24 | return type === 'Person' || type === 'Application' || type === 'Group' || type === 'Service' || type === 'Organization' | ||
25 | } | ||
26 | |||
27 | function isActorPublicKeyValid (publicKey: string) { | ||
28 | return exists(publicKey) && | ||
29 | typeof publicKey === 'string' && | ||
30 | publicKey.startsWith('-----BEGIN PUBLIC KEY-----') && | ||
31 | publicKey.includes('-----END PUBLIC KEY-----') && | ||
32 | validator.isLength(publicKey, CONSTRAINTS_FIELDS.ACTORS.PUBLIC_KEY) | ||
33 | } | ||
34 | |||
35 | const actorNameAlphabet = '[ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789\\-_.:]' | ||
36 | const actorNameRegExp = new RegExp(`^${actorNameAlphabet}+$`) | ||
37 | function isActorPreferredUsernameValid (preferredUsername: string) { | ||
38 | return exists(preferredUsername) && validator.matches(preferredUsername, actorNameRegExp) | ||
39 | } | ||
40 | |||
41 | function isActorPrivateKeyValid (privateKey: string) { | ||
42 | return exists(privateKey) && | ||
43 | typeof privateKey === 'string' && | ||
44 | (privateKey.startsWith('-----BEGIN RSA PRIVATE KEY-----') || privateKey.startsWith('-----BEGIN PRIVATE KEY-----')) && | ||
45 | // Sometimes there is a \n at the end, so just assert the string contains the end mark | ||
46 | (privateKey.includes('-----END RSA PRIVATE KEY-----') || privateKey.includes('-----END PRIVATE KEY-----')) && | ||
47 | validator.isLength(privateKey, CONSTRAINTS_FIELDS.ACTORS.PRIVATE_KEY) | ||
48 | } | ||
49 | |||
50 | function isActorFollowingCountValid (value: string) { | ||
51 | return exists(value) && validator.isInt('' + value, { min: 0 }) | ||
52 | } | ||
53 | |||
54 | function isActorFollowersCountValid (value: string) { | ||
55 | return exists(value) && validator.isInt('' + value, { min: 0 }) | ||
56 | } | ||
57 | |||
58 | function isActorDeleteActivityValid (activity: any) { | ||
59 | return isBaseActivityValid(activity, 'Delete') | ||
60 | } | ||
61 | |||
62 | function sanitizeAndCheckActorObject (actor: any) { | ||
63 | if (!isActorTypeValid(actor.type)) return false | ||
64 | |||
65 | normalizeActor(actor) | ||
66 | |||
67 | return exists(actor) && | ||
68 | isActivityPubUrlValid(actor.id) && | ||
69 | isActivityPubUrlValid(actor.inbox) && | ||
70 | isActorPreferredUsernameValid(actor.preferredUsername) && | ||
71 | isActivityPubUrlValid(actor.url) && | ||
72 | isActorPublicKeyObjectValid(actor.publicKey) && | ||
73 | isActorEndpointsObjectValid(actor.endpoints) && | ||
74 | |||
75 | (!actor.outbox || isActivityPubUrlValid(actor.outbox)) && | ||
76 | (!actor.following || isActivityPubUrlValid(actor.following)) && | ||
77 | (!actor.followers || isActivityPubUrlValid(actor.followers)) && | ||
78 | |||
79 | setValidAttributedTo(actor) && | ||
80 | setValidDescription(actor) && | ||
81 | // If this is a group (a channel), it should be attributed to an account | ||
82 | // In PeerTube we use this to attach a video channel to a specific account | ||
83 | (actor.type !== 'Group' || actor.attributedTo.length !== 0) | ||
84 | } | ||
85 | |||
86 | function normalizeActor (actor: any) { | ||
87 | if (!actor) return | ||
88 | |||
89 | if (!actor.url) { | ||
90 | actor.url = actor.id | ||
91 | } else if (typeof actor.url !== 'string') { | ||
92 | actor.url = actor.url.href || actor.url.url | ||
93 | } | ||
94 | |||
95 | if (!isDateValid(actor.published)) actor.published = undefined | ||
96 | |||
97 | if (actor.summary && typeof actor.summary === 'string') { | ||
98 | actor.summary = peertubeTruncate(actor.summary, { length: CONSTRAINTS_FIELDS.USERS.DESCRIPTION.max }) | ||
99 | |||
100 | if (actor.summary.length < CONSTRAINTS_FIELDS.USERS.DESCRIPTION.min) { | ||
101 | actor.summary = null | ||
102 | } | ||
103 | } | ||
104 | } | ||
105 | |||
106 | function isValidActorHandle (handle: string) { | ||
107 | if (!exists(handle)) return false | ||
108 | |||
109 | const parts = handle.split('@') | ||
110 | if (parts.length !== 2) return false | ||
111 | |||
112 | return isHostValid(parts[1]) | ||
113 | } | ||
114 | |||
115 | function areValidActorHandles (handles: string[]) { | ||
116 | return isArray(handles) && handles.every(h => isValidActorHandle(h)) | ||
117 | } | ||
118 | |||
119 | function setValidDescription (obj: any) { | ||
120 | if (!obj.summary) obj.summary = null | ||
121 | |||
122 | return true | ||
123 | } | ||
124 | |||
125 | // --------------------------------------------------------------------------- | ||
126 | |||
127 | export { | ||
128 | normalizeActor, | ||
129 | actorNameAlphabet, | ||
130 | areValidActorHandles, | ||
131 | isActorEndpointsObjectValid, | ||
132 | isActorPublicKeyObjectValid, | ||
133 | isActorTypeValid, | ||
134 | isActorPublicKeyValid, | ||
135 | isActorPreferredUsernameValid, | ||
136 | isActorPrivateKeyValid, | ||
137 | isActorFollowingCountValid, | ||
138 | isActorFollowersCountValid, | ||
139 | isActorDeleteActivityValid, | ||
140 | sanitizeAndCheckActorObject, | ||
141 | isValidActorHandle | ||
142 | } | ||
diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts deleted file mode 100644 index c5b3b4d9f..000000000 --- a/server/helpers/custom-validators/activitypub/cache-file.ts +++ /dev/null | |||
@@ -1,26 +0,0 @@ | |||
1 | import { isActivityPubUrlValid } from './misc' | ||
2 | import { isRemoteVideoUrlValid } from './videos' | ||
3 | import { exists, isDateValid } from '../misc' | ||
4 | import { CacheFileObject } from '../../../../shared/models/activitypub/objects' | ||
5 | |||
6 | function isCacheFileObjectValid (object: CacheFileObject) { | ||
7 | return exists(object) && | ||
8 | object.type === 'CacheFile' && | ||
9 | (object.expires === null || isDateValid(object.expires)) && | ||
10 | isActivityPubUrlValid(object.object) && | ||
11 | (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url)) | ||
12 | } | ||
13 | |||
14 | // --------------------------------------------------------------------------- | ||
15 | |||
16 | export { | ||
17 | isCacheFileObjectValid | ||
18 | } | ||
19 | |||
20 | // --------------------------------------------------------------------------- | ||
21 | |||
22 | function isPlaylistRedundancyUrlValid (url: any) { | ||
23 | return url.type === 'Link' && | ||
24 | (url.mediaType || url.mimeType) === 'application/x-mpegURL' && | ||
25 | isActivityPubUrlValid(url.href) | ||
26 | } | ||
diff --git a/server/helpers/custom-validators/activitypub/misc.ts b/server/helpers/custom-validators/activitypub/misc.ts deleted file mode 100644 index 7df47cf15..000000000 --- a/server/helpers/custom-validators/activitypub/misc.ts +++ /dev/null | |||
@@ -1,76 +0,0 @@ | |||
1 | import validator from 'validator' | ||
2 | import { CONFIG } from '@server/initializers/config' | ||
3 | import { CONSTRAINTS_FIELDS } from '../../../initializers/constants' | ||
4 | import { exists } from '../misc' | ||
5 | |||
6 | function isUrlValid (url: string) { | ||
7 | const isURLOptions = { | ||
8 | require_host: true, | ||
9 | require_tld: true, | ||
10 | require_protocol: true, | ||
11 | require_valid_protocol: true, | ||
12 | protocols: [ 'http', 'https' ] | ||
13 | } | ||
14 | |||
15 | // We validate 'localhost', so we don't have the top level domain | ||
16 | if (CONFIG.WEBSERVER.HOSTNAME === 'localhost' || CONFIG.WEBSERVER.HOSTNAME === '127.0.0.1') { | ||
17 | isURLOptions.require_tld = false | ||
18 | } | ||
19 | |||
20 | return exists(url) && validator.isURL('' + url, isURLOptions) | ||
21 | } | ||
22 | |||
23 | function isActivityPubUrlValid (url: string) { | ||
24 | return isUrlValid(url) && validator.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL) | ||
25 | } | ||
26 | |||
27 | function isBaseActivityValid (activity: any, type: string) { | ||
28 | return activity.type === type && | ||
29 | isActivityPubUrlValid(activity.id) && | ||
30 | isObjectValid(activity.actor) && | ||
31 | isUrlCollectionValid(activity.to) && | ||
32 | isUrlCollectionValid(activity.cc) | ||
33 | } | ||
34 | |||
35 | function isUrlCollectionValid (collection: any) { | ||
36 | return collection === undefined || | ||
37 | (Array.isArray(collection) && collection.every(t => isActivityPubUrlValid(t))) | ||
38 | } | ||
39 | |||
40 | function isObjectValid (object: any) { | ||
41 | return exists(object) && | ||
42 | ( | ||
43 | isActivityPubUrlValid(object) || isActivityPubUrlValid(object.id) | ||
44 | ) | ||
45 | } | ||
46 | |||
47 | function setValidAttributedTo (obj: any) { | ||
48 | if (Array.isArray(obj.attributedTo) === false) { | ||
49 | obj.attributedTo = [] | ||
50 | return true | ||
51 | } | ||
52 | |||
53 | obj.attributedTo = obj.attributedTo.filter(a => { | ||
54 | return isActivityPubUrlValid(a) || | ||
55 | ((a.type === 'Group' || a.type === 'Person') && isActivityPubUrlValid(a.id)) | ||
56 | }) | ||
57 | |||
58 | return true | ||
59 | } | ||
60 | |||
61 | function isActivityPubVideoDurationValid (value: string) { | ||
62 | // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration | ||
63 | return exists(value) && | ||
64 | typeof value === 'string' && | ||
65 | value.startsWith('PT') && | ||
66 | value.endsWith('S') | ||
67 | } | ||
68 | |||
69 | export { | ||
70 | isUrlValid, | ||
71 | isActivityPubUrlValid, | ||
72 | isBaseActivityValid, | ||
73 | setValidAttributedTo, | ||
74 | isObjectValid, | ||
75 | isActivityPubVideoDurationValid | ||
76 | } | ||
diff --git a/server/helpers/custom-validators/activitypub/playlist.ts b/server/helpers/custom-validators/activitypub/playlist.ts deleted file mode 100644 index 49bcadcfd..000000000 --- a/server/helpers/custom-validators/activitypub/playlist.ts +++ /dev/null | |||
@@ -1,29 +0,0 @@ | |||
1 | import validator from 'validator' | ||
2 | import { PlaylistElementObject, PlaylistObject } from '@shared/models' | ||
3 | import { exists, isDateValid, isUUIDValid } from '../misc' | ||
4 | import { isVideoPlaylistNameValid } from '../video-playlists' | ||
5 | import { isActivityPubUrlValid } from './misc' | ||
6 | |||
7 | function isPlaylistObjectValid (object: PlaylistObject) { | ||
8 | return exists(object) && | ||
9 | object.type === 'Playlist' && | ||
10 | validator.isInt(object.totalItems + '') && | ||
11 | isVideoPlaylistNameValid(object.name) && | ||
12 | isUUIDValid(object.uuid) && | ||
13 | isDateValid(object.published) && | ||
14 | isDateValid(object.updated) | ||
15 | } | ||
16 | |||
17 | function isPlaylistElementObjectValid (object: PlaylistElementObject) { | ||
18 | return exists(object) && | ||
19 | object.type === 'PlaylistElement' && | ||
20 | validator.isInt(object.position + '') && | ||
21 | isActivityPubUrlValid(object.url) | ||
22 | } | ||
23 | |||
24 | // --------------------------------------------------------------------------- | ||
25 | |||
26 | export { | ||
27 | isPlaylistObjectValid, | ||
28 | isPlaylistElementObjectValid | ||
29 | } | ||
diff --git a/server/helpers/custom-validators/activitypub/signature.ts b/server/helpers/custom-validators/activitypub/signature.ts deleted file mode 100644 index cfb65361e..000000000 --- a/server/helpers/custom-validators/activitypub/signature.ts +++ /dev/null | |||
@@ -1,22 +0,0 @@ | |||
1 | import { exists } from '../misc' | ||
2 | import { isActivityPubUrlValid } from './misc' | ||
3 | |||
4 | function isSignatureTypeValid (signatureType: string) { | ||
5 | return exists(signatureType) && signatureType === 'RsaSignature2017' | ||
6 | } | ||
7 | |||
8 | function isSignatureCreatorValid (signatureCreator: string) { | ||
9 | return exists(signatureCreator) && isActivityPubUrlValid(signatureCreator) | ||
10 | } | ||
11 | |||
12 | function isSignatureValueValid (signatureValue: string) { | ||
13 | return exists(signatureValue) && signatureValue.length > 0 | ||
14 | } | ||
15 | |||
16 | // --------------------------------------------------------------------------- | ||
17 | |||
18 | export { | ||
19 | isSignatureTypeValid, | ||
20 | isSignatureCreatorValid, | ||
21 | isSignatureValueValid | ||
22 | } | ||
diff --git a/server/helpers/custom-validators/activitypub/video-comments.ts b/server/helpers/custom-validators/activitypub/video-comments.ts deleted file mode 100644 index ea852c491..000000000 --- a/server/helpers/custom-validators/activitypub/video-comments.ts +++ /dev/null | |||
@@ -1,59 +0,0 @@ | |||
1 | import validator from 'validator' | ||
2 | import { ACTIVITY_PUB } from '../../../initializers/constants' | ||
3 | import { exists, isArray, isDateValid } from '../misc' | ||
4 | import { isActivityPubUrlValid } from './misc' | ||
5 | |||
6 | function sanitizeAndCheckVideoCommentObject (comment: any) { | ||
7 | if (!comment) return false | ||
8 | |||
9 | if (!isCommentTypeValid(comment)) return false | ||
10 | |||
11 | normalizeComment(comment) | ||
12 | |||
13 | if (comment.type === 'Tombstone') { | ||
14 | return isActivityPubUrlValid(comment.id) && | ||
15 | isDateValid(comment.published) && | ||
16 | isDateValid(comment.deleted) && | ||
17 | isActivityPubUrlValid(comment.url) | ||
18 | } | ||
19 | |||
20 | return isActivityPubUrlValid(comment.id) && | ||
21 | isCommentContentValid(comment.content) && | ||
22 | isActivityPubUrlValid(comment.inReplyTo) && | ||
23 | isDateValid(comment.published) && | ||
24 | isActivityPubUrlValid(comment.url) && | ||
25 | isArray(comment.to) && | ||
26 | ( | ||
27 | comment.to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 || | ||
28 | comment.cc.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 | ||
29 | ) // Only accept public comments | ||
30 | } | ||
31 | |||
32 | // --------------------------------------------------------------------------- | ||
33 | |||
34 | export { | ||
35 | sanitizeAndCheckVideoCommentObject | ||
36 | } | ||
37 | |||
38 | // --------------------------------------------------------------------------- | ||
39 | |||
40 | function isCommentContentValid (content: any) { | ||
41 | return exists(content) && validator.isLength('' + content, { min: 1 }) | ||
42 | } | ||
43 | |||
44 | function normalizeComment (comment: any) { | ||
45 | if (!comment) return | ||
46 | |||
47 | if (typeof comment.url !== 'string') { | ||
48 | if (typeof comment.url === 'object') comment.url = comment.url.href || comment.url.url | ||
49 | else comment.url = comment.id | ||
50 | } | ||
51 | } | ||
52 | |||
53 | function isCommentTypeValid (comment: any): boolean { | ||
54 | if (comment.type === 'Note') return true | ||
55 | |||
56 | if (comment.type === 'Tombstone' && comment.formerType === 'Note') return true | ||
57 | |||
58 | return false | ||
59 | } | ||
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts deleted file mode 100644 index 07e25b8ba..000000000 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ /dev/null | |||
@@ -1,241 +0,0 @@ | |||
1 | import validator from 'validator' | ||
2 | import { logger } from '@server/helpers/logger' | ||
3 | import { ActivityPubStoryboard, ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject, VideoObject } from '@shared/models' | ||
4 | import { LiveVideoLatencyMode, VideoState } from '../../../../shared/models/videos' | ||
5 | import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers/constants' | ||
6 | import { peertubeTruncate } from '../../core-utils' | ||
7 | import { isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc' | ||
8 | import { isLiveLatencyModeValid } from '../video-lives' | ||
9 | import { | ||
10 | isVideoDescriptionValid, | ||
11 | isVideoDurationValid, | ||
12 | isVideoNameValid, | ||
13 | isVideoStateValid, | ||
14 | isVideoTagValid, | ||
15 | isVideoViewsValid | ||
16 | } from '../videos' | ||
17 | import { isActivityPubUrlValid, isActivityPubVideoDurationValid, isBaseActivityValid, setValidAttributedTo } from './misc' | ||
18 | |||
19 | function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { | ||
20 | return isBaseActivityValid(activity, 'Update') && | ||
21 | sanitizeAndCheckVideoTorrentObject(activity.object) | ||
22 | } | ||
23 | |||
24 | function sanitizeAndCheckVideoTorrentObject (video: any) { | ||
25 | if (!video || video.type !== 'Video') return false | ||
26 | |||
27 | if (!setValidRemoteTags(video)) { | ||
28 | logger.debug('Video has invalid tags', { video }) | ||
29 | return false | ||
30 | } | ||
31 | if (!setValidRemoteVideoUrls(video)) { | ||
32 | logger.debug('Video has invalid urls', { video }) | ||
33 | return false | ||
34 | } | ||
35 | if (!setRemoteVideoContent(video)) { | ||
36 | logger.debug('Video has invalid content', { video }) | ||
37 | return false | ||
38 | } | ||
39 | if (!setValidAttributedTo(video)) { | ||
40 | logger.debug('Video has invalid attributedTo', { video }) | ||
41 | return false | ||
42 | } | ||
43 | if (!setValidRemoteCaptions(video)) { | ||
44 | logger.debug('Video has invalid captions', { video }) | ||
45 | return false | ||
46 | } | ||
47 | if (!setValidRemoteIcon(video)) { | ||
48 | logger.debug('Video has invalid icons', { video }) | ||
49 | return false | ||
50 | } | ||
51 | if (!setValidStoryboard(video)) { | ||
52 | logger.debug('Video has invalid preview (storyboard)', { video }) | ||
53 | return false | ||
54 | } | ||
55 | |||
56 | // Default attributes | ||
57 | if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED | ||
58 | if (!isBooleanValid(video.waitTranscoding)) video.waitTranscoding = false | ||
59 | if (!isBooleanValid(video.downloadEnabled)) video.downloadEnabled = true | ||
60 | if (!isBooleanValid(video.commentsEnabled)) video.commentsEnabled = false | ||
61 | if (!isBooleanValid(video.isLiveBroadcast)) video.isLiveBroadcast = false | ||
62 | if (!isBooleanValid(video.liveSaveReplay)) video.liveSaveReplay = false | ||
63 | if (!isBooleanValid(video.permanentLive)) video.permanentLive = false | ||
64 | if (!isLiveLatencyModeValid(video.latencyMode)) video.latencyMode = LiveVideoLatencyMode.DEFAULT | ||
65 | |||
66 | return isActivityPubUrlValid(video.id) && | ||
67 | isVideoNameValid(video.name) && | ||
68 | isActivityPubVideoDurationValid(video.duration) && | ||
69 | isVideoDurationValid(video.duration.replace(/[^0-9]+/g, '')) && | ||
70 | isUUIDValid(video.uuid) && | ||
71 | (!video.category || isRemoteNumberIdentifierValid(video.category)) && | ||
72 | (!video.licence || isRemoteNumberIdentifierValid(video.licence)) && | ||
73 | (!video.language || isRemoteStringIdentifierValid(video.language)) && | ||
74 | isVideoViewsValid(video.views) && | ||
75 | isBooleanValid(video.sensitive) && | ||
76 | isDateValid(video.published) && | ||
77 | isDateValid(video.updated) && | ||
78 | (!video.originallyPublishedAt || isDateValid(video.originallyPublishedAt)) && | ||
79 | (!video.uploadDate || isDateValid(video.uploadDate)) && | ||
80 | (!video.content || isRemoteVideoContentValid(video.mediaType, video.content)) && | ||
81 | video.attributedTo.length !== 0 | ||
82 | } | ||
83 | |||
84 | function isRemoteVideoUrlValid (url: any) { | ||
85 | return url.type === 'Link' && | ||
86 | // Video file link | ||
87 | ( | ||
88 | ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.includes(url.mediaType) && | ||
89 | isActivityPubUrlValid(url.href) && | ||
90 | validator.isInt(url.height + '', { min: 0 }) && | ||
91 | validator.isInt(url.size + '', { min: 0 }) && | ||
92 | (!url.fps || validator.isInt(url.fps + '', { min: -1 })) | ||
93 | ) || | ||
94 | // Torrent link | ||
95 | ( | ||
96 | ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.includes(url.mediaType) && | ||
97 | isActivityPubUrlValid(url.href) && | ||
98 | validator.isInt(url.height + '', { min: 0 }) | ||
99 | ) || | ||
100 | // Magnet link | ||
101 | ( | ||
102 | ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.includes(url.mediaType) && | ||
103 | validator.isLength(url.href, { min: 5 }) && | ||
104 | validator.isInt(url.height + '', { min: 0 }) | ||
105 | ) || | ||
106 | // HLS playlist link | ||
107 | ( | ||
108 | (url.mediaType || url.mimeType) === 'application/x-mpegURL' && | ||
109 | isActivityPubUrlValid(url.href) && | ||
110 | isArray(url.tag) | ||
111 | ) || | ||
112 | isAPVideoTrackerUrlObject(url) || | ||
113 | isAPVideoFileUrlMetadataObject(url) | ||
114 | } | ||
115 | |||
116 | function isAPVideoFileUrlMetadataObject (url: any): url is ActivityVideoFileMetadataUrlObject { | ||
117 | return url && | ||
118 | url.type === 'Link' && | ||
119 | url.mediaType === 'application/json' && | ||
120 | isArray(url.rel) && url.rel.includes('metadata') | ||
121 | } | ||
122 | |||
123 | function isAPVideoTrackerUrlObject (url: any): url is ActivityTrackerUrlObject { | ||
124 | return isArray(url.rel) && | ||
125 | url.rel.includes('tracker') && | ||
126 | isActivityPubUrlValid(url.href) | ||
127 | } | ||
128 | |||
129 | // --------------------------------------------------------------------------- | ||
130 | |||
131 | export { | ||
132 | sanitizeAndCheckVideoTorrentUpdateActivity, | ||
133 | isRemoteStringIdentifierValid, | ||
134 | sanitizeAndCheckVideoTorrentObject, | ||
135 | isRemoteVideoUrlValid, | ||
136 | isAPVideoFileUrlMetadataObject, | ||
137 | isAPVideoTrackerUrlObject | ||
138 | } | ||
139 | |||
140 | // --------------------------------------------------------------------------- | ||
141 | |||
142 | function setValidRemoteTags (video: any) { | ||
143 | if (Array.isArray(video.tag) === false) return false | ||
144 | |||
145 | video.tag = video.tag.filter(t => { | ||
146 | return t.type === 'Hashtag' && | ||
147 | isVideoTagValid(t.name) | ||
148 | }) | ||
149 | |||
150 | return true | ||
151 | } | ||
152 | |||
153 | function setValidRemoteCaptions (video: any) { | ||
154 | if (!video.subtitleLanguage) video.subtitleLanguage = [] | ||
155 | |||
156 | if (Array.isArray(video.subtitleLanguage) === false) return false | ||
157 | |||
158 | video.subtitleLanguage = video.subtitleLanguage.filter(caption => { | ||
159 | if (!isActivityPubUrlValid(caption.url)) caption.url = null | ||
160 | |||
161 | return isRemoteStringIdentifierValid(caption) | ||
162 | }) | ||
163 | |||
164 | return true | ||
165 | } | ||
166 | |||
167 | function isRemoteNumberIdentifierValid (data: any) { | ||
168 | return validator.isInt(data.identifier, { min: 0 }) | ||
169 | } | ||
170 | |||
171 | function isRemoteStringIdentifierValid (data: any) { | ||
172 | return typeof data.identifier === 'string' | ||
173 | } | ||
174 | |||
175 | function isRemoteVideoContentValid (mediaType: string, content: string) { | ||
176 | return mediaType === 'text/markdown' && isVideoDescriptionValid(content) | ||
177 | } | ||
178 | |||
179 | function setValidRemoteIcon (video: any) { | ||
180 | if (video.icon && !isArray(video.icon)) video.icon = [ video.icon ] | ||
181 | if (!video.icon) video.icon = [] | ||
182 | |||
183 | video.icon = video.icon.filter(icon => { | ||
184 | return icon.type === 'Image' && | ||
185 | isActivityPubUrlValid(icon.url) && | ||
186 | icon.mediaType === 'image/jpeg' && | ||
187 | validator.isInt(icon.width + '', { min: 0 }) && | ||
188 | validator.isInt(icon.height + '', { min: 0 }) | ||
189 | }) | ||
190 | |||
191 | return video.icon.length !== 0 | ||
192 | } | ||
193 | |||
194 | function setValidRemoteVideoUrls (video: any) { | ||
195 | if (Array.isArray(video.url) === false) return false | ||
196 | |||
197 | video.url = video.url.filter(u => isRemoteVideoUrlValid(u)) | ||
198 | |||
199 | return true | ||
200 | } | ||
201 | |||
202 | function setRemoteVideoContent (video: any) { | ||
203 | if (video.content) { | ||
204 | video.content = peertubeTruncate(video.content, { length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max }) | ||
205 | } | ||
206 | |||
207 | return true | ||
208 | } | ||
209 | |||
210 | function setValidStoryboard (video: VideoObject) { | ||
211 | if (!video.preview) return true | ||
212 | if (!Array.isArray(video.preview)) return false | ||
213 | |||
214 | video.preview = video.preview.filter(p => isStorybordValid(p)) | ||
215 | |||
216 | return true | ||
217 | } | ||
218 | |||
219 | function isStorybordValid (preview: ActivityPubStoryboard) { | ||
220 | if (!preview) return false | ||
221 | |||
222 | if ( | ||
223 | preview.type !== 'Image' || | ||
224 | !isArray(preview.rel) || | ||
225 | !preview.rel.includes('storyboard') | ||
226 | ) { | ||
227 | return false | ||
228 | } | ||
229 | |||
230 | preview.url = preview.url.filter(u => { | ||
231 | return u.mediaType === 'image/jpeg' && | ||
232 | isActivityPubUrlValid(u.href) && | ||
233 | validator.isInt(u.width + '', { min: 0 }) && | ||
234 | validator.isInt(u.height + '', { min: 0 }) && | ||
235 | validator.isInt(u.tileWidth + '', { min: 0 }) && | ||
236 | validator.isInt(u.tileHeight + '', { min: 0 }) && | ||
237 | isActivityPubVideoDurationValid(u.tileDuration) | ||
238 | }) | ||
239 | |||
240 | return preview.url.length !== 0 | ||
241 | } | ||
diff --git a/server/helpers/custom-validators/activitypub/watch-action.ts b/server/helpers/custom-validators/activitypub/watch-action.ts deleted file mode 100644 index b9ffa63f6..000000000 --- a/server/helpers/custom-validators/activitypub/watch-action.ts +++ /dev/null | |||
@@ -1,37 +0,0 @@ | |||
1 | import { WatchActionObject } from '@shared/models' | ||
2 | import { exists, isDateValid, isUUIDValid } from '../misc' | ||
3 | import { isVideoTimeValid } from '../video-view' | ||
4 | import { isActivityPubVideoDurationValid, isObjectValid } from './misc' | ||
5 | |||
6 | function isWatchActionObjectValid (action: WatchActionObject) { | ||
7 | return exists(action) && | ||
8 | action.type === 'WatchAction' && | ||
9 | isObjectValid(action.id) && | ||
10 | isActivityPubVideoDurationValid(action.duration) && | ||
11 | isDateValid(action.startTime) && | ||
12 | isDateValid(action.endTime) && | ||
13 | isLocationValid(action.location) && | ||
14 | isUUIDValid(action.uuid) && | ||
15 | isObjectValid(action.object) && | ||
16 | isWatchSectionsValid(action.watchSections) | ||
17 | } | ||
18 | |||
19 | // --------------------------------------------------------------------------- | ||
20 | |||
21 | export { | ||
22 | isWatchActionObjectValid | ||
23 | } | ||
24 | |||
25 | // --------------------------------------------------------------------------- | ||
26 | |||
27 | function isLocationValid (location: any) { | ||
28 | if (!location) return true | ||
29 | |||
30 | return typeof location === 'object' && typeof location.addressCountry === 'string' | ||
31 | } | ||
32 | |||
33 | function isWatchSectionsValid (sections: WatchActionObject['watchSections']) { | ||
34 | return Array.isArray(sections) && sections.every(s => { | ||
35 | return isVideoTimeValid(s.startTimestamp) && isVideoTimeValid(s.endTimestamp) | ||
36 | }) | ||
37 | } | ||
diff --git a/server/helpers/custom-validators/actor-images.ts b/server/helpers/custom-validators/actor-images.ts deleted file mode 100644 index 89f5a2262..000000000 --- a/server/helpers/custom-validators/actor-images.ts +++ /dev/null | |||
@@ -1,24 +0,0 @@ | |||
1 | |||
2 | import { UploadFilesForCheck } from 'express' | ||
3 | import { CONSTRAINTS_FIELDS } from '../../initializers/constants' | ||
4 | import { isFileValid } from './misc' | ||
5 | |||
6 | const imageMimeTypes = CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME | ||
7 | .map(v => v.replace('.', '')) | ||
8 | .join('|') | ||
9 | const imageMimeTypesRegex = `image/(${imageMimeTypes})` | ||
10 | |||
11 | function isActorImageFile (files: UploadFilesForCheck, fieldname: string) { | ||
12 | return isFileValid({ | ||
13 | files, | ||
14 | mimeTypeRegex: imageMimeTypesRegex, | ||
15 | field: fieldname, | ||
16 | maxSize: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max | ||
17 | }) | ||
18 | } | ||
19 | |||
20 | // --------------------------------------------------------------------------- | ||
21 | |||
22 | export { | ||
23 | isActorImageFile | ||
24 | } | ||
diff --git a/server/helpers/custom-validators/bulk.ts b/server/helpers/custom-validators/bulk.ts deleted file mode 100644 index 9e0ce0be1..000000000 --- a/server/helpers/custom-validators/bulk.ts +++ /dev/null | |||
@@ -1,9 +0,0 @@ | |||
1 | function isBulkRemoveCommentsOfScopeValid (value: string) { | ||
2 | return value === 'my-videos' || value === 'instance' | ||
3 | } | ||
4 | |||
5 | // --------------------------------------------------------------------------- | ||
6 | |||
7 | export { | ||
8 | isBulkRemoveCommentsOfScopeValid | ||
9 | } | ||
diff --git a/server/helpers/custom-validators/feeds.ts b/server/helpers/custom-validators/feeds.ts deleted file mode 100644 index fa35a7da6..000000000 --- a/server/helpers/custom-validators/feeds.ts +++ /dev/null | |||
@@ -1,23 +0,0 @@ | |||
1 | import { exists } from './misc' | ||
2 | |||
3 | function isValidRSSFeed (value: string) { | ||
4 | if (!exists(value)) return false | ||
5 | |||
6 | const feedExtensions = [ | ||
7 | 'xml', | ||
8 | 'json', | ||
9 | 'json1', | ||
10 | 'rss', | ||
11 | 'rss2', | ||
12 | 'atom', | ||
13 | 'atom1' | ||
14 | ] | ||
15 | |||
16 | return feedExtensions.includes(value) | ||
17 | } | ||
18 | |||
19 | // --------------------------------------------------------------------------- | ||
20 | |||
21 | export { | ||
22 | isValidRSSFeed | ||
23 | } | ||
diff --git a/server/helpers/custom-validators/follows.ts b/server/helpers/custom-validators/follows.ts deleted file mode 100644 index 0bec683c1..000000000 --- a/server/helpers/custom-validators/follows.ts +++ /dev/null | |||
@@ -1,30 +0,0 @@ | |||
1 | import { exists, isArray } from './misc' | ||
2 | import { FollowState } from '@shared/models' | ||
3 | |||
4 | function isFollowStateValid (value: FollowState) { | ||
5 | if (!exists(value)) return false | ||
6 | |||
7 | return value === 'pending' || value === 'accepted' || value === 'rejected' | ||
8 | } | ||
9 | |||
10 | function isRemoteHandleValid (value: string) { | ||
11 | if (!exists(value)) return false | ||
12 | if (typeof value !== 'string') return false | ||
13 | |||
14 | return value.includes('@') | ||
15 | } | ||
16 | |||
17 | function isEachUniqueHandleValid (handles: string[]) { | ||
18 | return isArray(handles) && | ||
19 | handles.every(handle => { | ||
20 | return isRemoteHandleValid(handle) && handles.indexOf(handle) === handles.lastIndexOf(handle) | ||
21 | }) | ||
22 | } | ||
23 | |||
24 | // --------------------------------------------------------------------------- | ||
25 | |||
26 | export { | ||
27 | isFollowStateValid, | ||
28 | isRemoteHandleValid, | ||
29 | isEachUniqueHandleValid | ||
30 | } | ||
diff --git a/server/helpers/custom-validators/jobs.ts b/server/helpers/custom-validators/jobs.ts deleted file mode 100644 index c168b3e91..000000000 --- a/server/helpers/custom-validators/jobs.ts +++ /dev/null | |||
@@ -1,21 +0,0 @@ | |||
1 | import { JobState } from '../../../shared/models' | ||
2 | import { exists } from './misc' | ||
3 | import { jobTypes } from '@server/lib/job-queue/job-queue' | ||
4 | |||
5 | const jobStates: JobState[] = [ 'active', 'completed', 'failed', 'waiting', 'delayed', 'paused', 'waiting-children' ] | ||
6 | |||
7 | function isValidJobState (value: JobState) { | ||
8 | return exists(value) && jobStates.includes(value) | ||
9 | } | ||
10 | |||
11 | function isValidJobType (value: any) { | ||
12 | return exists(value) && jobTypes.includes(value) | ||
13 | } | ||
14 | |||
15 | // --------------------------------------------------------------------------- | ||
16 | |||
17 | export { | ||
18 | jobStates, | ||
19 | isValidJobState, | ||
20 | isValidJobType | ||
21 | } | ||
diff --git a/server/helpers/custom-validators/logs.ts b/server/helpers/custom-validators/logs.ts deleted file mode 100644 index 215dbb0e1..000000000 --- a/server/helpers/custom-validators/logs.ts +++ /dev/null | |||
@@ -1,42 +0,0 @@ | |||
1 | import validator from 'validator' | ||
2 | import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' | ||
3 | import { ClientLogLevel, ServerLogLevel } from '@shared/models' | ||
4 | import { exists } from './misc' | ||
5 | |||
6 | const serverLogLevels = new Set<ServerLogLevel>([ 'debug', 'info', 'warn', 'error' ]) | ||
7 | const clientLogLevels = new Set<ClientLogLevel>([ 'warn', 'error' ]) | ||
8 | |||
9 | function isValidLogLevel (value: any) { | ||
10 | return exists(value) && serverLogLevels.has(value) | ||
11 | } | ||
12 | |||
13 | function isValidClientLogMessage (value: any) { | ||
14 | return typeof value === 'string' && validator.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_MESSAGE) | ||
15 | } | ||
16 | |||
17 | function isValidClientLogLevel (value: any) { | ||
18 | return exists(value) && clientLogLevels.has(value) | ||
19 | } | ||
20 | |||
21 | function isValidClientLogStackTrace (value: any) { | ||
22 | return typeof value === 'string' && validator.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_STACK_TRACE) | ||
23 | } | ||
24 | |||
25 | function isValidClientLogMeta (value: any) { | ||
26 | return typeof value === 'string' && validator.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_META) | ||
27 | } | ||
28 | |||
29 | function isValidClientLogUserAgent (value: any) { | ||
30 | return typeof value === 'string' && validator.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_USER_AGENT) | ||
31 | } | ||
32 | |||
33 | // --------------------------------------------------------------------------- | ||
34 | |||
35 | export { | ||
36 | isValidLogLevel, | ||
37 | isValidClientLogMessage, | ||
38 | isValidClientLogStackTrace, | ||
39 | isValidClientLogMeta, | ||
40 | isValidClientLogLevel, | ||
41 | isValidClientLogUserAgent | ||
42 | } | ||
diff --git a/server/helpers/custom-validators/metrics.ts b/server/helpers/custom-validators/metrics.ts deleted file mode 100644 index 44a863630..000000000 --- a/server/helpers/custom-validators/metrics.ts +++ /dev/null | |||
@@ -1,10 +0,0 @@ | |||
1 | function isValidPlayerMode (value: any) { | ||
2 | // TODO: remove webtorrent in v7 | ||
3 | return value === 'webtorrent' || value === 'web-video' || value === 'p2p-media-loader' | ||
4 | } | ||
5 | |||
6 | // --------------------------------------------------------------------------- | ||
7 | |||
8 | export { | ||
9 | isValidPlayerMode | ||
10 | } | ||
diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts deleted file mode 100644 index 937ae0632..000000000 --- a/server/helpers/custom-validators/misc.ts +++ /dev/null | |||
@@ -1,190 +0,0 @@ | |||
1 | import 'multer' | ||
2 | import { UploadFilesForCheck } from 'express' | ||
3 | import { sep } from 'path' | ||
4 | import validator from 'validator' | ||
5 | import { isShortUUID, shortToUUID } from '@shared/extra-utils' | ||
6 | |||
7 | function exists (value: any) { | ||
8 | return value !== undefined && value !== null | ||
9 | } | ||
10 | |||
11 | function isSafePath (p: string) { | ||
12 | return exists(p) && | ||
13 | (p + '').split(sep).every(part => { | ||
14 | return [ '..' ].includes(part) === false | ||
15 | }) | ||
16 | } | ||
17 | |||
18 | function isSafeFilename (filename: string, extension?: string) { | ||
19 | const regex = extension | ||
20 | ? new RegExp(`^[a-z0-9-]+\\.${extension}$`) | ||
21 | : new RegExp(`^[a-z0-9-]+\\.[a-z0-9]{1,8}$`) | ||
22 | |||
23 | return typeof filename === 'string' && !!filename.match(regex) | ||
24 | } | ||
25 | |||
26 | function isSafePeerTubeFilenameWithoutExtension (filename: string) { | ||
27 | return filename.match(/^[a-z0-9-]+$/) | ||
28 | } | ||
29 | |||
30 | function isArray (value: any): value is any[] { | ||
31 | return Array.isArray(value) | ||
32 | } | ||
33 | |||
34 | function isNotEmptyIntArray (value: any) { | ||
35 | return Array.isArray(value) && value.every(v => validator.isInt('' + v)) && value.length !== 0 | ||
36 | } | ||
37 | |||
38 | function isNotEmptyStringArray (value: any) { | ||
39 | return Array.isArray(value) && value.every(v => typeof v === 'string' && v.length !== 0) && value.length !== 0 | ||
40 | } | ||
41 | |||
42 | function isArrayOf (value: any, validator: (value: any) => boolean) { | ||
43 | return isArray(value) && value.every(v => validator(v)) | ||
44 | } | ||
45 | |||
46 | function isDateValid (value: string) { | ||
47 | return exists(value) && validator.isISO8601(value) | ||
48 | } | ||
49 | |||
50 | function isIdValid (value: string) { | ||
51 | return exists(value) && validator.isInt('' + value) | ||
52 | } | ||
53 | |||
54 | function isUUIDValid (value: string) { | ||
55 | return exists(value) && validator.isUUID('' + value, 4) | ||
56 | } | ||
57 | |||
58 | function areUUIDsValid (values: string[]) { | ||
59 | return isArray(values) && values.every(v => isUUIDValid(v)) | ||
60 | } | ||
61 | |||
62 | function isIdOrUUIDValid (value: string) { | ||
63 | return isIdValid(value) || isUUIDValid(value) | ||
64 | } | ||
65 | |||
66 | function isBooleanValid (value: any) { | ||
67 | return typeof value === 'boolean' || (typeof value === 'string' && validator.isBoolean(value)) | ||
68 | } | ||
69 | |||
70 | function isIntOrNull (value: any) { | ||
71 | return value === null || validator.isInt('' + value) | ||
72 | } | ||
73 | |||
74 | // --------------------------------------------------------------------------- | ||
75 | |||
76 | function isFileValid (options: { | ||
77 | files: UploadFilesForCheck | ||
78 | |||
79 | maxSize: number | null | ||
80 | mimeTypeRegex: string | null | ||
81 | |||
82 | field?: string | ||
83 | |||
84 | optional?: boolean // Default false | ||
85 | }) { | ||
86 | const { files, mimeTypeRegex, field, maxSize, optional = false } = options | ||
87 | |||
88 | // Should have files | ||
89 | if (!files) return optional | ||
90 | |||
91 | const fileArray = isArray(files) | ||
92 | ? files | ||
93 | : files[field] | ||
94 | |||
95 | if (!fileArray || !isArray(fileArray) || fileArray.length === 0) { | ||
96 | return optional | ||
97 | } | ||
98 | |||
99 | // The file exists | ||
100 | const file = fileArray[0] | ||
101 | if (!file?.originalname) return false | ||
102 | |||
103 | // Check size | ||
104 | if ((maxSize !== null) && file.size > maxSize) return false | ||
105 | |||
106 | if (mimeTypeRegex === null) return true | ||
107 | |||
108 | return checkMimetypeRegex(file.mimetype, mimeTypeRegex) | ||
109 | } | ||
110 | |||
111 | function checkMimetypeRegex (fileMimeType: string, mimeTypeRegex: string) { | ||
112 | return new RegExp(`^${mimeTypeRegex}$`, 'i').test(fileMimeType) | ||
113 | } | ||
114 | |||
115 | // --------------------------------------------------------------------------- | ||
116 | |||
117 | function toCompleteUUID (value: string) { | ||
118 | if (isShortUUID(value)) { | ||
119 | try { | ||
120 | return shortToUUID(value) | ||
121 | } catch { | ||
122 | return '' | ||
123 | } | ||
124 | } | ||
125 | |||
126 | return value | ||
127 | } | ||
128 | |||
129 | function toCompleteUUIDs (values: string[]) { | ||
130 | return values.map(v => toCompleteUUID(v)) | ||
131 | } | ||
132 | |||
133 | function toIntOrNull (value: string) { | ||
134 | const v = toValueOrNull(value) | ||
135 | |||
136 | if (v === null || v === undefined) return v | ||
137 | if (typeof v === 'number') return v | ||
138 | |||
139 | return validator.toInt('' + v) | ||
140 | } | ||
141 | |||
142 | function toBooleanOrNull (value: any) { | ||
143 | const v = toValueOrNull(value) | ||
144 | |||
145 | if (v === null || v === undefined) return v | ||
146 | if (typeof v === 'boolean') return v | ||
147 | |||
148 | return validator.toBoolean('' + v) | ||
149 | } | ||
150 | |||
151 | function toValueOrNull (value: string) { | ||
152 | if (value === 'null') return null | ||
153 | |||
154 | return value | ||
155 | } | ||
156 | |||
157 | function toIntArray (value: any) { | ||
158 | if (!value) return [] | ||
159 | if (isArray(value) === false) return [ validator.toInt(value) ] | ||
160 | |||
161 | return value.map(v => validator.toInt(v)) | ||
162 | } | ||
163 | |||
164 | // --------------------------------------------------------------------------- | ||
165 | |||
166 | export { | ||
167 | exists, | ||
168 | isArrayOf, | ||
169 | isNotEmptyIntArray, | ||
170 | isArray, | ||
171 | isIntOrNull, | ||
172 | isIdValid, | ||
173 | isSafePath, | ||
174 | isNotEmptyStringArray, | ||
175 | isUUIDValid, | ||
176 | toCompleteUUIDs, | ||
177 | toCompleteUUID, | ||
178 | isIdOrUUIDValid, | ||
179 | isDateValid, | ||
180 | toValueOrNull, | ||
181 | toBooleanOrNull, | ||
182 | isBooleanValid, | ||
183 | toIntOrNull, | ||
184 | areUUIDsValid, | ||
185 | toIntArray, | ||
186 | isFileValid, | ||
187 | isSafePeerTubeFilenameWithoutExtension, | ||
188 | isSafeFilename, | ||
189 | checkMimetypeRegex | ||
190 | } | ||
diff --git a/server/helpers/custom-validators/plugins.ts b/server/helpers/custom-validators/plugins.ts deleted file mode 100644 index a20de0c4a..000000000 --- a/server/helpers/custom-validators/plugins.ts +++ /dev/null | |||
@@ -1,178 +0,0 @@ | |||
1 | import validator from 'validator' | ||
2 | import { PluginPackageJSON } from '../../../shared/models/plugins/plugin-package-json.model' | ||
3 | import { PluginType } from '../../../shared/models/plugins/plugin.type' | ||
4 | import { CONSTRAINTS_FIELDS } from '../../initializers/constants' | ||
5 | import { isUrlValid } from './activitypub/misc' | ||
6 | import { exists, isArray, isSafePath } from './misc' | ||
7 | |||
8 | const PLUGINS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.PLUGINS | ||
9 | |||
10 | function isPluginTypeValid (value: any) { | ||
11 | return exists(value) && | ||
12 | (value === PluginType.PLUGIN || value === PluginType.THEME) | ||
13 | } | ||
14 | |||
15 | function isPluginNameValid (value: string) { | ||
16 | return exists(value) && | ||
17 | validator.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.NAME) && | ||
18 | validator.matches(value, /^[a-z-0-9]+$/) | ||
19 | } | ||
20 | |||
21 | function isNpmPluginNameValid (value: string) { | ||
22 | return exists(value) && | ||
23 | validator.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.NAME) && | ||
24 | validator.matches(value, /^[a-z\-._0-9]+$/) && | ||
25 | (value.startsWith('peertube-plugin-') || value.startsWith('peertube-theme-')) | ||
26 | } | ||
27 | |||
28 | function isPluginDescriptionValid (value: string) { | ||
29 | return exists(value) && validator.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.DESCRIPTION) | ||
30 | } | ||
31 | |||
32 | function isPluginStableVersionValid (value: string) { | ||
33 | if (!exists(value)) return false | ||
34 | |||
35 | const parts = (value + '').split('.') | ||
36 | |||
37 | return parts.length === 3 && parts.every(p => validator.isInt(p)) | ||
38 | } | ||
39 | |||
40 | function isPluginStableOrUnstableVersionValid (value: string) { | ||
41 | if (!exists(value)) return false | ||
42 | |||
43 | // suffix is beta.x or alpha.x | ||
44 | const [ stable, suffix ] = value.split('-') | ||
45 | if (!isPluginStableVersionValid(stable)) return false | ||
46 | |||
47 | const suffixRegex = /^(rc|alpha|beta)\.\d+$/ | ||
48 | if (suffix && !suffixRegex.test(suffix)) return false | ||
49 | |||
50 | return true | ||
51 | } | ||
52 | |||
53 | function isPluginEngineValid (engine: any) { | ||
54 | return exists(engine) && exists(engine.peertube) | ||
55 | } | ||
56 | |||
57 | function isPluginHomepage (value: string) { | ||
58 | return exists(value) && (!value || isUrlValid(value)) | ||
59 | } | ||
60 | |||
61 | function isPluginBugs (value: string) { | ||
62 | return exists(value) && (!value || isUrlValid(value)) | ||
63 | } | ||
64 | |||
65 | function areStaticDirectoriesValid (staticDirs: any) { | ||
66 | if (!exists(staticDirs) || typeof staticDirs !== 'object') return false | ||
67 | |||
68 | for (const key of Object.keys(staticDirs)) { | ||
69 | if (!isSafePath(staticDirs[key])) return false | ||
70 | } | ||
71 | |||
72 | return true | ||
73 | } | ||
74 | |||
75 | function areClientScriptsValid (clientScripts: any[]) { | ||
76 | return isArray(clientScripts) && | ||
77 | clientScripts.every(c => { | ||
78 | return isSafePath(c.script) && isArray(c.scopes) | ||
79 | }) | ||
80 | } | ||
81 | |||
82 | function areTranslationPathsValid (translations: any) { | ||
83 | if (!exists(translations) || typeof translations !== 'object') return false | ||
84 | |||
85 | for (const key of Object.keys(translations)) { | ||
86 | if (!isSafePath(translations[key])) return false | ||
87 | } | ||
88 | |||
89 | return true | ||
90 | } | ||
91 | |||
92 | function areCSSPathsValid (css: any[]) { | ||
93 | return isArray(css) && css.every(c => isSafePath(c)) | ||
94 | } | ||
95 | |||
96 | function isThemeNameValid (name: string) { | ||
97 | return isPluginNameValid(name) | ||
98 | } | ||
99 | |||
100 | function isPackageJSONValid (packageJSON: PluginPackageJSON, pluginType: PluginType) { | ||
101 | let result = true | ||
102 | const badFields: string[] = [] | ||
103 | |||
104 | if (!isNpmPluginNameValid(packageJSON.name)) { | ||
105 | result = false | ||
106 | badFields.push('name') | ||
107 | } | ||
108 | |||
109 | if (!isPluginDescriptionValid(packageJSON.description)) { | ||
110 | result = false | ||
111 | badFields.push('description') | ||
112 | } | ||
113 | |||
114 | if (!isPluginEngineValid(packageJSON.engine)) { | ||
115 | result = false | ||
116 | badFields.push('engine') | ||
117 | } | ||
118 | |||
119 | if (!isPluginHomepage(packageJSON.homepage)) { | ||
120 | result = false | ||
121 | badFields.push('homepage') | ||
122 | } | ||
123 | |||
124 | if (!exists(packageJSON.author)) { | ||
125 | result = false | ||
126 | badFields.push('author') | ||
127 | } | ||
128 | |||
129 | if (!isPluginBugs(packageJSON.bugs)) { | ||
130 | result = false | ||
131 | badFields.push('bugs') | ||
132 | } | ||
133 | |||
134 | if (pluginType === PluginType.PLUGIN && !isSafePath(packageJSON.library)) { | ||
135 | result = false | ||
136 | badFields.push('library') | ||
137 | } | ||
138 | |||
139 | if (!areStaticDirectoriesValid(packageJSON.staticDirs)) { | ||
140 | result = false | ||
141 | badFields.push('staticDirs') | ||
142 | } | ||
143 | |||
144 | if (!areCSSPathsValid(packageJSON.css)) { | ||
145 | result = false | ||
146 | badFields.push('css') | ||
147 | } | ||
148 | |||
149 | if (!areClientScriptsValid(packageJSON.clientScripts)) { | ||
150 | result = false | ||
151 | badFields.push('clientScripts') | ||
152 | } | ||
153 | |||
154 | if (!areTranslationPathsValid(packageJSON.translations)) { | ||
155 | result = false | ||
156 | badFields.push('translations') | ||
157 | } | ||
158 | |||
159 | return { result, badFields } | ||
160 | } | ||
161 | |||
162 | function isLibraryCodeValid (library: any) { | ||
163 | return typeof library.register === 'function' && | ||
164 | typeof library.unregister === 'function' | ||
165 | } | ||
166 | |||
167 | export { | ||
168 | isPluginTypeValid, | ||
169 | isPackageJSONValid, | ||
170 | isThemeNameValid, | ||
171 | isPluginHomepage, | ||
172 | isPluginStableVersionValid, | ||
173 | isPluginStableOrUnstableVersionValid, | ||
174 | isPluginNameValid, | ||
175 | isPluginDescriptionValid, | ||
176 | isLibraryCodeValid, | ||
177 | isNpmPluginNameValid | ||
178 | } | ||
diff --git a/server/helpers/custom-validators/runners/jobs.ts b/server/helpers/custom-validators/runners/jobs.ts deleted file mode 100644 index 6349e79ba..000000000 --- a/server/helpers/custom-validators/runners/jobs.ts +++ /dev/null | |||
@@ -1,197 +0,0 @@ | |||
1 | import { UploadFilesForCheck } from 'express' | ||
2 | import validator from 'validator' | ||
3 | import { CONSTRAINTS_FIELDS, RUNNER_JOB_STATES } from '@server/initializers/constants' | ||
4 | import { | ||
5 | LiveRTMPHLSTranscodingSuccess, | ||
6 | RunnerJobSuccessPayload, | ||
7 | RunnerJobType, | ||
8 | RunnerJobUpdatePayload, | ||
9 | VideoStudioTranscodingSuccess, | ||
10 | VODAudioMergeTranscodingSuccess, | ||
11 | VODHLSTranscodingSuccess, | ||
12 | VODWebVideoTranscodingSuccess | ||
13 | } from '@shared/models' | ||
14 | import { exists, isArray, isFileValid, isSafeFilename } from '../misc' | ||
15 | |||
16 | const RUNNER_JOBS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNER_JOBS | ||
17 | |||
18 | const runnerJobTypes = new Set([ 'vod-hls-transcoding', 'vod-web-video-transcoding', 'vod-audio-merge-transcoding' ]) | ||
19 | function isRunnerJobTypeValid (value: RunnerJobType) { | ||
20 | return runnerJobTypes.has(value) | ||
21 | } | ||
22 | |||
23 | function isRunnerJobSuccessPayloadValid (value: RunnerJobSuccessPayload, type: RunnerJobType, files: UploadFilesForCheck) { | ||
24 | return isRunnerJobVODWebVideoResultPayloadValid(value as VODWebVideoTranscodingSuccess, type, files) || | ||
25 | isRunnerJobVODHLSResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) || | ||
26 | isRunnerJobVODAudioMergeResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) || | ||
27 | isRunnerJobLiveRTMPHLSResultPayloadValid(value as LiveRTMPHLSTranscodingSuccess, type) || | ||
28 | isRunnerJobVideoStudioResultPayloadValid(value as VideoStudioTranscodingSuccess, type, files) | ||
29 | } | ||
30 | |||
31 | // --------------------------------------------------------------------------- | ||
32 | |||
33 | function isRunnerJobProgressValid (value: string) { | ||
34 | return validator.isInt(value + '', RUNNER_JOBS_CONSTRAINTS_FIELDS.PROGRESS) | ||
35 | } | ||
36 | |||
37 | function isRunnerJobUpdatePayloadValid (value: RunnerJobUpdatePayload, type: RunnerJobType, files: UploadFilesForCheck) { | ||
38 | return isRunnerJobVODWebVideoUpdatePayloadValid(value, type, files) || | ||
39 | isRunnerJobVODHLSUpdatePayloadValid(value, type, files) || | ||
40 | isRunnerJobVideoStudioUpdatePayloadValid(value, type, files) || | ||
41 | isRunnerJobVODAudioMergeUpdatePayloadValid(value, type, files) || | ||
42 | isRunnerJobLiveRTMPHLSUpdatePayloadValid(value, type, files) | ||
43 | } | ||
44 | |||
45 | // --------------------------------------------------------------------------- | ||
46 | |||
47 | function isRunnerJobTokenValid (value: string) { | ||
48 | return exists(value) && validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.TOKEN) | ||
49 | } | ||
50 | |||
51 | function isRunnerJobAbortReasonValid (value: string) { | ||
52 | return validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.REASON) | ||
53 | } | ||
54 | |||
55 | function isRunnerJobErrorMessageValid (value: string) { | ||
56 | return validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.ERROR_MESSAGE) | ||
57 | } | ||
58 | |||
59 | function isRunnerJobStateValid (value: any) { | ||
60 | return exists(value) && RUNNER_JOB_STATES[value] !== undefined | ||
61 | } | ||
62 | |||
63 | function isRunnerJobArrayOfStateValid (value: any) { | ||
64 | return isArray(value) && value.every(v => isRunnerJobStateValid(v)) | ||
65 | } | ||
66 | |||
67 | // --------------------------------------------------------------------------- | ||
68 | |||
69 | export { | ||
70 | isRunnerJobTypeValid, | ||
71 | isRunnerJobSuccessPayloadValid, | ||
72 | isRunnerJobUpdatePayloadValid, | ||
73 | isRunnerJobTokenValid, | ||
74 | isRunnerJobErrorMessageValid, | ||
75 | isRunnerJobProgressValid, | ||
76 | isRunnerJobAbortReasonValid, | ||
77 | isRunnerJobArrayOfStateValid, | ||
78 | isRunnerJobStateValid | ||
79 | } | ||
80 | |||
81 | // --------------------------------------------------------------------------- | ||
82 | |||
83 | function isRunnerJobVODWebVideoResultPayloadValid ( | ||
84 | _value: VODWebVideoTranscodingSuccess, | ||
85 | type: RunnerJobType, | ||
86 | files: UploadFilesForCheck | ||
87 | ) { | ||
88 | return type === 'vod-web-video-transcoding' && | ||
89 | isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) | ||
90 | } | ||
91 | |||
92 | function isRunnerJobVODHLSResultPayloadValid ( | ||
93 | _value: VODHLSTranscodingSuccess, | ||
94 | type: RunnerJobType, | ||
95 | files: UploadFilesForCheck | ||
96 | ) { | ||
97 | return type === 'vod-hls-transcoding' && | ||
98 | isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) && | ||
99 | isFileValid({ files, field: 'payload[resolutionPlaylistFile]', mimeTypeRegex: null, maxSize: null }) | ||
100 | } | ||
101 | |||
102 | function isRunnerJobVODAudioMergeResultPayloadValid ( | ||
103 | _value: VODAudioMergeTranscodingSuccess, | ||
104 | type: RunnerJobType, | ||
105 | files: UploadFilesForCheck | ||
106 | ) { | ||
107 | return type === 'vod-audio-merge-transcoding' && | ||
108 | isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) | ||
109 | } | ||
110 | |||
111 | function isRunnerJobLiveRTMPHLSResultPayloadValid ( | ||
112 | value: LiveRTMPHLSTranscodingSuccess, | ||
113 | type: RunnerJobType | ||
114 | ) { | ||
115 | return type === 'live-rtmp-hls-transcoding' && (!value || (typeof value === 'object' && Object.keys(value).length === 0)) | ||
116 | } | ||
117 | |||
118 | function isRunnerJobVideoStudioResultPayloadValid ( | ||
119 | _value: VideoStudioTranscodingSuccess, | ||
120 | type: RunnerJobType, | ||
121 | files: UploadFilesForCheck | ||
122 | ) { | ||
123 | return type === 'video-studio-transcoding' && | ||
124 | isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) | ||
125 | } | ||
126 | |||
127 | // --------------------------------------------------------------------------- | ||
128 | |||
129 | function isRunnerJobVODWebVideoUpdatePayloadValid ( | ||
130 | value: RunnerJobUpdatePayload, | ||
131 | type: RunnerJobType, | ||
132 | _files: UploadFilesForCheck | ||
133 | ) { | ||
134 | return type === 'vod-web-video-transcoding' && | ||
135 | (!value || (typeof value === 'object' && Object.keys(value).length === 0)) | ||
136 | } | ||
137 | |||
138 | function isRunnerJobVODHLSUpdatePayloadValid ( | ||
139 | value: RunnerJobUpdatePayload, | ||
140 | type: RunnerJobType, | ||
141 | _files: UploadFilesForCheck | ||
142 | ) { | ||
143 | return type === 'vod-hls-transcoding' && | ||
144 | (!value || (typeof value === 'object' && Object.keys(value).length === 0)) | ||
145 | } | ||
146 | |||
147 | function isRunnerJobVODAudioMergeUpdatePayloadValid ( | ||
148 | value: RunnerJobUpdatePayload, | ||
149 | type: RunnerJobType, | ||
150 | _files: UploadFilesForCheck | ||
151 | ) { | ||
152 | return type === 'vod-audio-merge-transcoding' && | ||
153 | (!value || (typeof value === 'object' && Object.keys(value).length === 0)) | ||
154 | } | ||
155 | |||
156 | function isRunnerJobLiveRTMPHLSUpdatePayloadValid ( | ||
157 | value: RunnerJobUpdatePayload, | ||
158 | type: RunnerJobType, | ||
159 | files: UploadFilesForCheck | ||
160 | ) { | ||
161 | let result = type === 'live-rtmp-hls-transcoding' && !!value && !!files | ||
162 | |||
163 | result &&= isFileValid({ files, field: 'payload[masterPlaylistFile]', mimeTypeRegex: null, maxSize: null, optional: true }) | ||
164 | |||
165 | result &&= isFileValid({ | ||
166 | files, | ||
167 | field: 'payload[resolutionPlaylistFile]', | ||
168 | mimeTypeRegex: null, | ||
169 | maxSize: null, | ||
170 | optional: !value.resolutionPlaylistFilename | ||
171 | }) | ||
172 | |||
173 | if (files['payload[resolutionPlaylistFile]']) { | ||
174 | result &&= isSafeFilename(value.resolutionPlaylistFilename, 'm3u8') | ||
175 | } | ||
176 | |||
177 | return result && | ||
178 | isSafeFilename(value.videoChunkFilename, 'ts') && | ||
179 | ( | ||
180 | ( | ||
181 | value.type === 'remove-chunk' | ||
182 | ) || | ||
183 | ( | ||
184 | value.type === 'add-chunk' && | ||
185 | isFileValid({ files, field: 'payload[videoChunkFile]', mimeTypeRegex: null, maxSize: null }) | ||
186 | ) | ||
187 | ) | ||
188 | } | ||
189 | |||
190 | function isRunnerJobVideoStudioUpdatePayloadValid ( | ||
191 | value: RunnerJobUpdatePayload, | ||
192 | type: RunnerJobType, | ||
193 | _files: UploadFilesForCheck | ||
194 | ) { | ||
195 | return type === 'video-studio-transcoding' && | ||
196 | (!value || (typeof value === 'object' && Object.keys(value).length === 0)) | ||
197 | } | ||
diff --git a/server/helpers/custom-validators/runners/runners.ts b/server/helpers/custom-validators/runners/runners.ts deleted file mode 100644 index 953fac3b5..000000000 --- a/server/helpers/custom-validators/runners/runners.ts +++ /dev/null | |||
@@ -1,30 +0,0 @@ | |||
1 | import validator from 'validator' | ||
2 | import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' | ||
3 | import { exists } from '../misc' | ||
4 | |||
5 | const RUNNERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNERS | ||
6 | |||
7 | function isRunnerRegistrationTokenValid (value: string) { | ||
8 | return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.TOKEN) | ||
9 | } | ||
10 | |||
11 | function isRunnerTokenValid (value: string) { | ||
12 | return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.TOKEN) | ||
13 | } | ||
14 | |||
15 | function isRunnerNameValid (value: string) { | ||
16 | return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.NAME) | ||
17 | } | ||
18 | |||
19 | function isRunnerDescriptionValid (value: string) { | ||
20 | return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.DESCRIPTION) | ||
21 | } | ||
22 | |||
23 | // --------------------------------------------------------------------------- | ||
24 | |||
25 | export { | ||
26 | isRunnerRegistrationTokenValid, | ||
27 | isRunnerTokenValid, | ||
28 | isRunnerNameValid, | ||
29 | isRunnerDescriptionValid | ||
30 | } | ||
diff --git a/server/helpers/custom-validators/search.ts b/server/helpers/custom-validators/search.ts deleted file mode 100644 index 6dba5d14e..000000000 --- a/server/helpers/custom-validators/search.ts +++ /dev/null | |||
@@ -1,37 +0,0 @@ | |||
1 | import validator from 'validator' | ||
2 | import { SearchTargetType } from '@shared/models/search/search-target-query.model' | ||
3 | import { isArray, exists } from './misc' | ||
4 | import { CONFIG } from '@server/initializers/config' | ||
5 | |||
6 | function isNumberArray (value: any) { | ||
7 | return isArray(value) && value.every(v => validator.isInt('' + v)) | ||
8 | } | ||
9 | |||
10 | function isStringArray (value: any) { | ||
11 | return isArray(value) && value.every(v => typeof v === 'string') | ||
12 | } | ||
13 | |||
14 | function isBooleanBothQueryValid (value: any) { | ||
15 | return value === 'true' || value === 'false' || value === 'both' | ||
16 | } | ||
17 | |||
18 | function isSearchTargetValid (value: SearchTargetType) { | ||
19 | if (!exists(value)) return true | ||
20 | |||
21 | const searchIndexConfig = CONFIG.SEARCH.SEARCH_INDEX | ||
22 | |||
23 | if (value === 'local') return true | ||
24 | |||
25 | if (value === 'search-index' && searchIndexConfig.ENABLED) return true | ||
26 | |||
27 | return false | ||
28 | } | ||
29 | |||
30 | // --------------------------------------------------------------------------- | ||
31 | |||
32 | export { | ||
33 | isNumberArray, | ||
34 | isStringArray, | ||
35 | isBooleanBothQueryValid, | ||
36 | isSearchTargetValid | ||
37 | } | ||
diff --git a/server/helpers/custom-validators/servers.ts b/server/helpers/custom-validators/servers.ts deleted file mode 100644 index b2aa03b77..000000000 --- a/server/helpers/custom-validators/servers.ts +++ /dev/null | |||
@@ -1,42 +0,0 @@ | |||
1 | import validator from 'validator' | ||
2 | import { CONFIG } from '@server/initializers/config' | ||
3 | import { CONSTRAINTS_FIELDS } from '../../initializers/constants' | ||
4 | import { exists, isArray } from './misc' | ||
5 | |||
6 | function isHostValid (host: string) { | ||
7 | const isURLOptions = { | ||
8 | require_host: true, | ||
9 | require_tld: true | ||
10 | } | ||
11 | |||
12 | // We validate 'localhost', so we don't have the top level domain | ||
13 | if (CONFIG.WEBSERVER.HOSTNAME === 'localhost' || CONFIG.WEBSERVER.HOSTNAME === '127.0.0.1') { | ||
14 | isURLOptions.require_tld = false | ||
15 | } | ||
16 | |||
17 | return exists(host) && validator.isURL(host, isURLOptions) && host.split('://').length === 1 | ||
18 | } | ||
19 | |||
20 | function isEachUniqueHostValid (hosts: string[]) { | ||
21 | return isArray(hosts) && | ||
22 | hosts.every(host => { | ||
23 | return isHostValid(host) && hosts.indexOf(host) === hosts.lastIndexOf(host) | ||
24 | }) | ||
25 | } | ||
26 | |||
27 | function isValidContactBody (value: any) { | ||
28 | return exists(value) && validator.isLength(value, CONSTRAINTS_FIELDS.CONTACT_FORM.BODY) | ||
29 | } | ||
30 | |||
31 | function isValidContactFromName (value: any) { | ||
32 | return exists(value) && validator.isLength(value, CONSTRAINTS_FIELDS.CONTACT_FORM.FROM_NAME) | ||
33 | } | ||
34 | |||
35 | // --------------------------------------------------------------------------- | ||
36 | |||
37 | export { | ||
38 | isValidContactBody, | ||
39 | isValidContactFromName, | ||
40 | isEachUniqueHostValid, | ||
41 | isHostValid | ||
42 | } | ||
diff --git a/server/helpers/custom-validators/user-notifications.ts b/server/helpers/custom-validators/user-notifications.ts deleted file mode 100644 index 2de13ca09..000000000 --- a/server/helpers/custom-validators/user-notifications.ts +++ /dev/null | |||
@@ -1,23 +0,0 @@ | |||
1 | import validator from 'validator' | ||
2 | import { UserNotificationSettingValue } from '@shared/models' | ||
3 | import { exists } from './misc' | ||
4 | |||
5 | function isUserNotificationTypeValid (value: any) { | ||
6 | return exists(value) && validator.isInt('' + value) | ||
7 | } | ||
8 | |||
9 | function isUserNotificationSettingValid (value: any) { | ||
10 | return exists(value) && | ||
11 | validator.isInt('' + value) && | ||
12 | ( | ||
13 | value === UserNotificationSettingValue.NONE || | ||
14 | value === UserNotificationSettingValue.WEB || | ||
15 | value === UserNotificationSettingValue.EMAIL || | ||
16 | value === (UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL) | ||
17 | ) | ||
18 | } | ||
19 | |||
20 | export { | ||
21 | isUserNotificationSettingValid, | ||
22 | isUserNotificationTypeValid | ||
23 | } | ||
diff --git a/server/helpers/custom-validators/user-registration.ts b/server/helpers/custom-validators/user-registration.ts deleted file mode 100644 index 9da0bb08a..000000000 --- a/server/helpers/custom-validators/user-registration.ts +++ /dev/null | |||
@@ -1,25 +0,0 @@ | |||
1 | import validator from 'validator' | ||
2 | import { CONSTRAINTS_FIELDS, USER_REGISTRATION_STATES } from '../../initializers/constants' | ||
3 | import { exists } from './misc' | ||
4 | |||
5 | const USER_REGISTRATIONS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.USER_REGISTRATIONS | ||
6 | |||
7 | function isRegistrationStateValid (value: string) { | ||
8 | return exists(value) && USER_REGISTRATION_STATES[value] !== undefined | ||
9 | } | ||
10 | |||
11 | function isRegistrationModerationResponseValid (value: string) { | ||
12 | return exists(value) && validator.isLength(value, USER_REGISTRATIONS_CONSTRAINTS_FIELDS.MODERATOR_MESSAGE) | ||
13 | } | ||
14 | |||
15 | function isRegistrationReasonValid (value: string) { | ||
16 | return exists(value) && validator.isLength(value, USER_REGISTRATIONS_CONSTRAINTS_FIELDS.REASON_MESSAGE) | ||
17 | } | ||
18 | |||
19 | // --------------------------------------------------------------------------- | ||
20 | |||
21 | export { | ||
22 | isRegistrationStateValid, | ||
23 | isRegistrationModerationResponseValid, | ||
24 | isRegistrationReasonValid | ||
25 | } | ||
diff --git a/server/helpers/custom-validators/users.ts b/server/helpers/custom-validators/users.ts deleted file mode 100644 index f02b3ba65..000000000 --- a/server/helpers/custom-validators/users.ts +++ /dev/null | |||
@@ -1,123 +0,0 @@ | |||
1 | import validator from 'validator' | ||
2 | import { UserRole } from '@shared/models' | ||
3 | import { isEmailEnabled } from '../../initializers/config' | ||
4 | import { CONSTRAINTS_FIELDS, NSFW_POLICY_TYPES } from '../../initializers/constants' | ||
5 | import { exists, isArray, isBooleanValid } from './misc' | ||
6 | |||
7 | const USERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.USERS | ||
8 | |||
9 | function isUserPasswordValid (value: string) { | ||
10 | return validator.isLength(value, USERS_CONSTRAINTS_FIELDS.PASSWORD) | ||
11 | } | ||
12 | |||
13 | function isUserPasswordValidOrEmpty (value: string) { | ||
14 | // Empty password is only possible if emailing is enabled. | ||
15 | if (value === '') return isEmailEnabled() | ||
16 | |||
17 | return isUserPasswordValid(value) | ||
18 | } | ||
19 | |||
20 | function isUserVideoQuotaValid (value: string) { | ||
21 | return exists(value) && validator.isInt(value + '', USERS_CONSTRAINTS_FIELDS.VIDEO_QUOTA) | ||
22 | } | ||
23 | |||
24 | function isUserVideoQuotaDailyValid (value: string) { | ||
25 | return exists(value) && validator.isInt(value + '', USERS_CONSTRAINTS_FIELDS.VIDEO_QUOTA_DAILY) | ||
26 | } | ||
27 | |||
28 | function isUserUsernameValid (value: string) { | ||
29 | return exists(value) && | ||
30 | validator.matches(value, new RegExp(`^[a-z0-9_]+([a-z0-9_.-]+[a-z0-9_]+)?$`)) && | ||
31 | validator.isLength(value, USERS_CONSTRAINTS_FIELDS.USERNAME) | ||
32 | } | ||
33 | |||
34 | function isUserDisplayNameValid (value: string) { | ||
35 | return value === null || (exists(value) && validator.isLength(value, CONSTRAINTS_FIELDS.USERS.NAME)) | ||
36 | } | ||
37 | |||
38 | function isUserDescriptionValid (value: string) { | ||
39 | return value === null || (exists(value) && validator.isLength(value, CONSTRAINTS_FIELDS.USERS.DESCRIPTION)) | ||
40 | } | ||
41 | |||
42 | function isUserEmailVerifiedValid (value: any) { | ||
43 | return isBooleanValid(value) | ||
44 | } | ||
45 | |||
46 | const nsfwPolicies = new Set(Object.values(NSFW_POLICY_TYPES)) | ||
47 | function isUserNSFWPolicyValid (value: any) { | ||
48 | return exists(value) && nsfwPolicies.has(value) | ||
49 | } | ||
50 | |||
51 | function isUserP2PEnabledValid (value: any) { | ||
52 | return isBooleanValid(value) | ||
53 | } | ||
54 | |||
55 | function isUserVideosHistoryEnabledValid (value: any) { | ||
56 | return isBooleanValid(value) | ||
57 | } | ||
58 | |||
59 | function isUserAutoPlayVideoValid (value: any) { | ||
60 | return isBooleanValid(value) | ||
61 | } | ||
62 | |||
63 | function isUserVideoLanguages (value: any) { | ||
64 | return value === null || (isArray(value) && value.length < CONSTRAINTS_FIELDS.USERS.VIDEO_LANGUAGES.max) | ||
65 | } | ||
66 | |||
67 | function isUserAdminFlagsValid (value: any) { | ||
68 | return exists(value) && validator.isInt('' + value) | ||
69 | } | ||
70 | |||
71 | function isUserBlockedValid (value: any) { | ||
72 | return isBooleanValid(value) | ||
73 | } | ||
74 | |||
75 | function isUserAutoPlayNextVideoValid (value: any) { | ||
76 | return isBooleanValid(value) | ||
77 | } | ||
78 | |||
79 | function isUserAutoPlayNextVideoPlaylistValid (value: any) { | ||
80 | return isBooleanValid(value) | ||
81 | } | ||
82 | |||
83 | function isUserEmailPublicValid (value: any) { | ||
84 | return isBooleanValid(value) | ||
85 | } | ||
86 | |||
87 | function isUserNoModal (value: any) { | ||
88 | return isBooleanValid(value) | ||
89 | } | ||
90 | |||
91 | function isUserBlockedReasonValid (value: any) { | ||
92 | return value === null || (exists(value) && validator.isLength(value, CONSTRAINTS_FIELDS.USERS.BLOCKED_REASON)) | ||
93 | } | ||
94 | |||
95 | function isUserRoleValid (value: any) { | ||
96 | return exists(value) && validator.isInt('' + value) && [ UserRole.ADMINISTRATOR, UserRole.MODERATOR, UserRole.USER ].includes(value) | ||
97 | } | ||
98 | |||
99 | // --------------------------------------------------------------------------- | ||
100 | |||
101 | export { | ||
102 | isUserVideosHistoryEnabledValid, | ||
103 | isUserBlockedValid, | ||
104 | isUserPasswordValid, | ||
105 | isUserPasswordValidOrEmpty, | ||
106 | isUserVideoLanguages, | ||
107 | isUserBlockedReasonValid, | ||
108 | isUserRoleValid, | ||
109 | isUserVideoQuotaValid, | ||
110 | isUserVideoQuotaDailyValid, | ||
111 | isUserUsernameValid, | ||
112 | isUserAdminFlagsValid, | ||
113 | isUserEmailVerifiedValid, | ||
114 | isUserNSFWPolicyValid, | ||
115 | isUserP2PEnabledValid, | ||
116 | isUserAutoPlayVideoValid, | ||
117 | isUserAutoPlayNextVideoValid, | ||
118 | isUserAutoPlayNextVideoPlaylistValid, | ||
119 | isUserDisplayNameValid, | ||
120 | isUserDescriptionValid, | ||
121 | isUserEmailPublicValid, | ||
122 | isUserNoModal | ||
123 | } | ||
diff --git a/server/helpers/custom-validators/video-blacklist.ts b/server/helpers/custom-validators/video-blacklist.ts deleted file mode 100644 index 34fcec38e..000000000 --- a/server/helpers/custom-validators/video-blacklist.ts +++ /dev/null | |||
@@ -1,22 +0,0 @@ | |||
1 | import validator from 'validator' | ||
2 | import { exists } from './misc' | ||
3 | import { CONSTRAINTS_FIELDS } from '../../initializers/constants' | ||
4 | import { VideoBlacklistType } from '../../../shared/models/videos' | ||
5 | |||
6 | const VIDEO_BLACKLIST_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_BLACKLIST | ||
7 | |||
8 | function isVideoBlacklistReasonValid (value: string) { | ||
9 | return value === null || validator.isLength(value, VIDEO_BLACKLIST_CONSTRAINTS_FIELDS.REASON) | ||
10 | } | ||
11 | |||
12 | function isVideoBlacklistTypeValid (value: any) { | ||
13 | return exists(value) && | ||
14 | (value === VideoBlacklistType.AUTO_BEFORE_PUBLISHED || value === VideoBlacklistType.MANUAL) | ||
15 | } | ||
16 | |||
17 | // --------------------------------------------------------------------------- | ||
18 | |||
19 | export { | ||
20 | isVideoBlacklistReasonValid, | ||
21 | isVideoBlacklistTypeValid | ||
22 | } | ||
diff --git a/server/helpers/custom-validators/video-captions.ts b/server/helpers/custom-validators/video-captions.ts deleted file mode 100644 index 0e24655a0..000000000 --- a/server/helpers/custom-validators/video-captions.ts +++ /dev/null | |||
@@ -1,43 +0,0 @@ | |||
1 | import { UploadFilesForCheck } from 'express' | ||
2 | import { readFile } from 'fs-extra' | ||
3 | import { getFileSize } from '@shared/extra-utils' | ||
4 | import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_LANGUAGES } from '../../initializers/constants' | ||
5 | import { logger } from '../logger' | ||
6 | import { exists, isFileValid } from './misc' | ||
7 | |||
8 | function isVideoCaptionLanguageValid (value: any) { | ||
9 | return exists(value) && VIDEO_LANGUAGES[value] !== undefined | ||
10 | } | ||
11 | |||
12 | // MacOS sends application/octet-stream | ||
13 | const videoCaptionTypesRegex = [ ...Object.keys(MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT), 'application/octet-stream' ] | ||
14 | .map(m => `(${m})`) | ||
15 | .join('|') | ||
16 | |||
17 | function isVideoCaptionFile (files: UploadFilesForCheck, field: string) { | ||
18 | return isFileValid({ | ||
19 | files, | ||
20 | mimeTypeRegex: videoCaptionTypesRegex, | ||
21 | field, | ||
22 | maxSize: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max | ||
23 | }) | ||
24 | } | ||
25 | |||
26 | async function isVTTFileValid (filePath: string) { | ||
27 | const size = await getFileSize(filePath) | ||
28 | const content = await readFile(filePath, 'utf8') | ||
29 | |||
30 | logger.debug('Checking VTT file %s', filePath, { size, content }) | ||
31 | |||
32 | if (size > CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max) return false | ||
33 | |||
34 | return content?.startsWith('WEBVTT') | ||
35 | } | ||
36 | |||
37 | // --------------------------------------------------------------------------- | ||
38 | |||
39 | export { | ||
40 | isVideoCaptionFile, | ||
41 | isVTTFileValid, | ||
42 | isVideoCaptionLanguageValid | ||
43 | } | ||
diff --git a/server/helpers/custom-validators/video-channel-syncs.ts b/server/helpers/custom-validators/video-channel-syncs.ts deleted file mode 100644 index c5a9afa96..000000000 --- a/server/helpers/custom-validators/video-channel-syncs.ts +++ /dev/null | |||
@@ -1,6 +0,0 @@ | |||
1 | import { VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants' | ||
2 | import { exists } from './misc' | ||
3 | |||
4 | export function isVideoChannelSyncStateValid (value: any) { | ||
5 | return exists(value) && VIDEO_CHANNEL_SYNC_STATE[value] !== undefined | ||
6 | } | ||
diff --git a/server/helpers/custom-validators/video-channels.ts b/server/helpers/custom-validators/video-channels.ts deleted file mode 100644 index 249083f39..000000000 --- a/server/helpers/custom-validators/video-channels.ts +++ /dev/null | |||
@@ -1,32 +0,0 @@ | |||
1 | import validator from 'validator' | ||
2 | import { CONSTRAINTS_FIELDS } from '../../initializers/constants' | ||
3 | import { exists } from './misc' | ||
4 | import { isUserUsernameValid } from './users' | ||
5 | |||
6 | const VIDEO_CHANNELS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_CHANNELS | ||
7 | |||
8 | function isVideoChannelUsernameValid (value: string) { | ||
9 | // Use the same constraints than user username | ||
10 | return isUserUsernameValid(value) | ||
11 | } | ||
12 | |||
13 | function isVideoChannelDescriptionValid (value: string) { | ||
14 | return value === null || validator.isLength(value, VIDEO_CHANNELS_CONSTRAINTS_FIELDS.DESCRIPTION) | ||
15 | } | ||
16 | |||
17 | function isVideoChannelDisplayNameValid (value: string) { | ||
18 | return exists(value) && validator.isLength(value, VIDEO_CHANNELS_CONSTRAINTS_FIELDS.NAME) | ||
19 | } | ||
20 | |||
21 | function isVideoChannelSupportValid (value: string) { | ||
22 | return value === null || (exists(value) && validator.isLength(value, VIDEO_CHANNELS_CONSTRAINTS_FIELDS.SUPPORT)) | ||
23 | } | ||
24 | |||
25 | // --------------------------------------------------------------------------- | ||
26 | |||
27 | export { | ||
28 | isVideoChannelUsernameValid, | ||
29 | isVideoChannelDescriptionValid, | ||
30 | isVideoChannelDisplayNameValid, | ||
31 | isVideoChannelSupportValid | ||
32 | } | ||
diff --git a/server/helpers/custom-validators/video-comments.ts b/server/helpers/custom-validators/video-comments.ts deleted file mode 100644 index 94bdf237a..000000000 --- a/server/helpers/custom-validators/video-comments.ts +++ /dev/null | |||
@@ -1,14 +0,0 @@ | |||
1 | import validator from 'validator' | ||
2 | import { CONSTRAINTS_FIELDS } from '../../initializers/constants' | ||
3 | |||
4 | const VIDEO_COMMENTS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_COMMENTS | ||
5 | |||
6 | function isValidVideoCommentText (value: string) { | ||
7 | return value === null || validator.isLength(value, VIDEO_COMMENTS_CONSTRAINTS_FIELDS.TEXT) | ||
8 | } | ||
9 | |||
10 | // --------------------------------------------------------------------------- | ||
11 | |||
12 | export { | ||
13 | isValidVideoCommentText | ||
14 | } | ||
diff --git a/server/helpers/custom-validators/video-imports.ts b/server/helpers/custom-validators/video-imports.ts deleted file mode 100644 index da8962cb6..000000000 --- a/server/helpers/custom-validators/video-imports.ts +++ /dev/null | |||
@@ -1,46 +0,0 @@ | |||
1 | import 'multer' | ||
2 | import { UploadFilesForCheck } from 'express' | ||
3 | import validator from 'validator' | ||
4 | import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_IMPORT_STATES } from '../../initializers/constants' | ||
5 | import { exists, isFileValid } from './misc' | ||
6 | |||
7 | function isVideoImportTargetUrlValid (url: string) { | ||
8 | const isURLOptions = { | ||
9 | require_host: true, | ||
10 | require_tld: true, | ||
11 | require_protocol: true, | ||
12 | require_valid_protocol: true, | ||
13 | protocols: [ 'http', 'https' ] | ||
14 | } | ||
15 | |||
16 | return exists(url) && | ||
17 | validator.isURL('' + url, isURLOptions) && | ||
18 | validator.isLength('' + url, CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL) | ||
19 | } | ||
20 | |||
21 | function isVideoImportStateValid (value: any) { | ||
22 | return exists(value) && VIDEO_IMPORT_STATES[value] !== undefined | ||
23 | } | ||
24 | |||
25 | // MacOS sends application/octet-stream | ||
26 | const videoTorrentImportRegex = [ ...Object.keys(MIMETYPES.TORRENT.MIMETYPE_EXT), 'application/octet-stream' ] | ||
27 | .map(m => `(${m})`) | ||
28 | .join('|') | ||
29 | |||
30 | function isVideoImportTorrentFile (files: UploadFilesForCheck) { | ||
31 | return isFileValid({ | ||
32 | files, | ||
33 | mimeTypeRegex: videoTorrentImportRegex, | ||
34 | field: 'torrentfile', | ||
35 | maxSize: CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_FILE.FILE_SIZE.max, | ||
36 | optional: true | ||
37 | }) | ||
38 | } | ||
39 | |||
40 | // --------------------------------------------------------------------------- | ||
41 | |||
42 | export { | ||
43 | isVideoImportStateValid, | ||
44 | isVideoImportTargetUrlValid, | ||
45 | isVideoImportTorrentFile | ||
46 | } | ||
diff --git a/server/helpers/custom-validators/video-lives.ts b/server/helpers/custom-validators/video-lives.ts deleted file mode 100644 index 69d08ae68..000000000 --- a/server/helpers/custom-validators/video-lives.ts +++ /dev/null | |||
@@ -1,11 +0,0 @@ | |||
1 | import { LiveVideoLatencyMode } from '@shared/models' | ||
2 | |||
3 | function isLiveLatencyModeValid (value: any) { | ||
4 | return [ LiveVideoLatencyMode.DEFAULT, LiveVideoLatencyMode.SMALL_LATENCY, LiveVideoLatencyMode.HIGH_LATENCY ].includes(value) | ||
5 | } | ||
6 | |||
7 | // --------------------------------------------------------------------------- | ||
8 | |||
9 | export { | ||
10 | isLiveLatencyModeValid | ||
11 | } | ||
diff --git a/server/helpers/custom-validators/video-ownership.ts b/server/helpers/custom-validators/video-ownership.ts deleted file mode 100644 index cf15b385a..000000000 --- a/server/helpers/custom-validators/video-ownership.ts +++ /dev/null | |||
@@ -1,20 +0,0 @@ | |||
1 | import { Response } from 'express' | ||
2 | import { MUserId } from '@server/types/models' | ||
3 | import { MVideoChangeOwnershipFull } from '@server/types/models/video/video-change-ownership' | ||
4 | import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' | ||
5 | |||
6 | function checkUserCanTerminateOwnershipChange (user: MUserId, videoChangeOwnership: MVideoChangeOwnershipFull, res: Response) { | ||
7 | if (videoChangeOwnership.NextOwner.userId === user.id) { | ||
8 | return true | ||
9 | } | ||
10 | |||
11 | res.fail({ | ||
12 | status: HttpStatusCode.FORBIDDEN_403, | ||
13 | message: 'Cannot terminate an ownership change of another user' | ||
14 | }) | ||
15 | return false | ||
16 | } | ||
17 | |||
18 | export { | ||
19 | checkUserCanTerminateOwnershipChange | ||
20 | } | ||
diff --git a/server/helpers/custom-validators/video-playlists.ts b/server/helpers/custom-validators/video-playlists.ts deleted file mode 100644 index 180018fc5..000000000 --- a/server/helpers/custom-validators/video-playlists.ts +++ /dev/null | |||
@@ -1,35 +0,0 @@ | |||
1 | import { exists } from './misc' | ||
2 | import validator from 'validator' | ||
3 | import { CONSTRAINTS_FIELDS, VIDEO_PLAYLIST_PRIVACIES, VIDEO_PLAYLIST_TYPES } from '../../initializers/constants' | ||
4 | |||
5 | const PLAYLISTS_CONSTRAINT_FIELDS = CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS | ||
6 | |||
7 | function isVideoPlaylistNameValid (value: any) { | ||
8 | return exists(value) && validator.isLength(value, PLAYLISTS_CONSTRAINT_FIELDS.NAME) | ||
9 | } | ||
10 | |||
11 | function isVideoPlaylistDescriptionValid (value: any) { | ||
12 | return value === null || (exists(value) && validator.isLength(value, PLAYLISTS_CONSTRAINT_FIELDS.DESCRIPTION)) | ||
13 | } | ||
14 | |||
15 | function isVideoPlaylistPrivacyValid (value: number) { | ||
16 | return validator.isInt(value + '') && VIDEO_PLAYLIST_PRIVACIES[value] !== undefined | ||
17 | } | ||
18 | |||
19 | function isVideoPlaylistTimestampValid (value: any) { | ||
20 | return value === null || (exists(value) && validator.isInt('' + value, { min: 0 })) | ||
21 | } | ||
22 | |||
23 | function isVideoPlaylistTypeValid (value: any) { | ||
24 | return exists(value) && VIDEO_PLAYLIST_TYPES[value] !== undefined | ||
25 | } | ||
26 | |||
27 | // --------------------------------------------------------------------------- | ||
28 | |||
29 | export { | ||
30 | isVideoPlaylistNameValid, | ||
31 | isVideoPlaylistDescriptionValid, | ||
32 | isVideoPlaylistPrivacyValid, | ||
33 | isVideoPlaylistTimestampValid, | ||
34 | isVideoPlaylistTypeValid | ||
35 | } | ||
diff --git a/server/helpers/custom-validators/video-rates.ts b/server/helpers/custom-validators/video-rates.ts deleted file mode 100644 index f2b6f7cae..000000000 --- a/server/helpers/custom-validators/video-rates.ts +++ /dev/null | |||
@@ -1,5 +0,0 @@ | |||
1 | function isRatingValid (value: any) { | ||
2 | return value === 'like' || value === 'dislike' | ||
3 | } | ||
4 | |||
5 | export { isRatingValid } | ||
diff --git a/server/helpers/custom-validators/video-redundancies.ts b/server/helpers/custom-validators/video-redundancies.ts deleted file mode 100644 index 50a559c4f..000000000 --- a/server/helpers/custom-validators/video-redundancies.ts +++ /dev/null | |||
@@ -1,12 +0,0 @@ | |||
1 | import { exists } from './misc' | ||
2 | |||
3 | function isVideoRedundancyTarget (value: any) { | ||
4 | return exists(value) && | ||
5 | (value === 'my-videos' || value === 'remote-videos') | ||
6 | } | ||
7 | |||
8 | // --------------------------------------------------------------------------- | ||
9 | |||
10 | export { | ||
11 | isVideoRedundancyTarget | ||
12 | } | ||
diff --git a/server/helpers/custom-validators/video-stats.ts b/server/helpers/custom-validators/video-stats.ts deleted file mode 100644 index 1e22f0654..000000000 --- a/server/helpers/custom-validators/video-stats.ts +++ /dev/null | |||
@@ -1,16 +0,0 @@ | |||
1 | import { VideoStatsTimeserieMetric } from '@shared/models' | ||
2 | |||
3 | const validMetrics = new Set<VideoStatsTimeserieMetric>([ | ||
4 | 'viewers', | ||
5 | 'aggregateWatchTime' | ||
6 | ]) | ||
7 | |||
8 | function isValidStatTimeserieMetric (value: VideoStatsTimeserieMetric) { | ||
9 | return validMetrics.has(value) | ||
10 | } | ||
11 | |||
12 | // --------------------------------------------------------------------------- | ||
13 | |||
14 | export { | ||
15 | isValidStatTimeserieMetric | ||
16 | } | ||
diff --git a/server/helpers/custom-validators/video-studio.ts b/server/helpers/custom-validators/video-studio.ts deleted file mode 100644 index 68dfec8dd..000000000 --- a/server/helpers/custom-validators/video-studio.ts +++ /dev/null | |||
@@ -1,53 +0,0 @@ | |||
1 | import validator from 'validator' | ||
2 | import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' | ||
3 | import { buildTaskFileFieldname } from '@server/lib/video-studio' | ||
4 | import { VideoStudioTask } from '@shared/models' | ||
5 | import { isArray } from './misc' | ||
6 | import { isVideoFileMimeTypeValid, isVideoImageValid } from './videos' | ||
7 | import { forceNumber } from '@shared/core-utils' | ||
8 | |||
9 | function isValidStudioTasksArray (tasks: any) { | ||
10 | if (!isArray(tasks)) return false | ||
11 | |||
12 | return tasks.length >= CONSTRAINTS_FIELDS.VIDEO_STUDIO.TASKS.min && | ||
13 | tasks.length <= CONSTRAINTS_FIELDS.VIDEO_STUDIO.TASKS.max | ||
14 | } | ||
15 | |||
16 | function isStudioCutTaskValid (task: VideoStudioTask) { | ||
17 | if (task.name !== 'cut') return false | ||
18 | if (!task.options) return false | ||
19 | |||
20 | const { start, end } = task.options | ||
21 | if (!start && !end) return false | ||
22 | |||
23 | if (start && !validator.isInt(start + '', CONSTRAINTS_FIELDS.VIDEO_STUDIO.CUT_TIME)) return false | ||
24 | if (end && !validator.isInt(end + '', CONSTRAINTS_FIELDS.VIDEO_STUDIO.CUT_TIME)) return false | ||
25 | |||
26 | if (!start || !end) return true | ||
27 | |||
28 | return forceNumber(start) < forceNumber(end) | ||
29 | } | ||
30 | |||
31 | function isStudioTaskAddIntroOutroValid (task: VideoStudioTask, indice: number, files: Express.Multer.File[]) { | ||
32 | const file = files.find(f => f.fieldname === buildTaskFileFieldname(indice, 'file')) | ||
33 | |||
34 | return (task.name === 'add-intro' || task.name === 'add-outro') && | ||
35 | file && isVideoFileMimeTypeValid([ file ], null) | ||
36 | } | ||
37 | |||
38 | function isStudioTaskAddWatermarkValid (task: VideoStudioTask, indice: number, files: Express.Multer.File[]) { | ||
39 | const file = files.find(f => f.fieldname === buildTaskFileFieldname(indice, 'file')) | ||
40 | |||
41 | return task.name === 'add-watermark' && | ||
42 | file && isVideoImageValid([ file ], null, true) | ||
43 | } | ||
44 | |||
45 | // --------------------------------------------------------------------------- | ||
46 | |||
47 | export { | ||
48 | isValidStudioTasksArray, | ||
49 | |||
50 | isStudioCutTaskValid, | ||
51 | isStudioTaskAddIntroOutroValid, | ||
52 | isStudioTaskAddWatermarkValid | ||
53 | } | ||
diff --git a/server/helpers/custom-validators/video-transcoding.ts b/server/helpers/custom-validators/video-transcoding.ts deleted file mode 100644 index 220530de4..000000000 --- a/server/helpers/custom-validators/video-transcoding.ts +++ /dev/null | |||
@@ -1,12 +0,0 @@ | |||
1 | import { exists } from './misc' | ||
2 | |||
3 | function isValidCreateTranscodingType (value: any) { | ||
4 | return exists(value) && | ||
5 | (value === 'hls' || value === 'webtorrent' || value === 'web-video') // TODO: remove webtorrent in v7 | ||
6 | } | ||
7 | |||
8 | // --------------------------------------------------------------------------- | ||
9 | |||
10 | export { | ||
11 | isValidCreateTranscodingType | ||
12 | } | ||
diff --git a/server/helpers/custom-validators/video-view.ts b/server/helpers/custom-validators/video-view.ts deleted file mode 100644 index 091c92083..000000000 --- a/server/helpers/custom-validators/video-view.ts +++ /dev/null | |||
@@ -1,12 +0,0 @@ | |||
1 | import { exists } from './misc' | ||
2 | |||
3 | function isVideoTimeValid (value: number, videoDuration?: number) { | ||
4 | if (value < 0) return false | ||
5 | if (exists(videoDuration) && value > videoDuration) return false | ||
6 | |||
7 | return true | ||
8 | } | ||
9 | |||
10 | export { | ||
11 | isVideoTimeValid | ||
12 | } | ||
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts deleted file mode 100644 index 00c6deed4..000000000 --- a/server/helpers/custom-validators/videos.ts +++ /dev/null | |||
@@ -1,218 +0,0 @@ | |||
1 | import { Request, Response, UploadFilesForCheck } from 'express' | ||
2 | import { decode as magnetUriDecode } from 'magnet-uri' | ||
3 | import validator from 'validator' | ||
4 | import { getVideoWithAttributes } from '@server/helpers/video' | ||
5 | import { HttpStatusCode, VideoInclude, VideoPrivacy, VideoRateType } from '@shared/models' | ||
6 | import { | ||
7 | CONSTRAINTS_FIELDS, | ||
8 | MIMETYPES, | ||
9 | VIDEO_CATEGORIES, | ||
10 | VIDEO_LICENCES, | ||
11 | VIDEO_LIVE, | ||
12 | VIDEO_PRIVACIES, | ||
13 | VIDEO_RATE_TYPES, | ||
14 | VIDEO_STATES | ||
15 | } from '../../initializers/constants' | ||
16 | import { exists, isArray, isDateValid, isFileValid } from './misc' | ||
17 | |||
18 | const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS | ||
19 | |||
20 | function isVideoIncludeValid (include: VideoInclude) { | ||
21 | return exists(include) && validator.isInt('' + include) | ||
22 | } | ||
23 | |||
24 | function isVideoCategoryValid (value: any) { | ||
25 | return value === null || VIDEO_CATEGORIES[value] !== undefined | ||
26 | } | ||
27 | |||
28 | function isVideoStateValid (value: any) { | ||
29 | return exists(value) && VIDEO_STATES[value] !== undefined | ||
30 | } | ||
31 | |||
32 | function isVideoLicenceValid (value: any) { | ||
33 | return value === null || VIDEO_LICENCES[value] !== undefined | ||
34 | } | ||
35 | |||
36 | function isVideoLanguageValid (value: any) { | ||
37 | return value === null || | ||
38 | (typeof value === 'string' && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.LANGUAGE)) | ||
39 | } | ||
40 | |||
41 | function isVideoDurationValid (value: string) { | ||
42 | return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION) | ||
43 | } | ||
44 | |||
45 | function isVideoDescriptionValid (value: string) { | ||
46 | return value === null || (exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.DESCRIPTION)) | ||
47 | } | ||
48 | |||
49 | function isVideoSupportValid (value: string) { | ||
50 | return value === null || (exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.SUPPORT)) | ||
51 | } | ||
52 | |||
53 | function isVideoNameValid (value: string) { | ||
54 | return exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.NAME) | ||
55 | } | ||
56 | |||
57 | function isVideoTagValid (tag: string) { | ||
58 | return exists(tag) && validator.isLength(tag, VIDEOS_CONSTRAINTS_FIELDS.TAG) | ||
59 | } | ||
60 | |||
61 | function areVideoTagsValid (tags: string[]) { | ||
62 | return tags === null || ( | ||
63 | isArray(tags) && | ||
64 | validator.isInt(tags.length.toString(), VIDEOS_CONSTRAINTS_FIELDS.TAGS) && | ||
65 | tags.every(tag => isVideoTagValid(tag)) | ||
66 | ) | ||
67 | } | ||
68 | |||
69 | function isVideoViewsValid (value: string) { | ||
70 | return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.VIEWS) | ||
71 | } | ||
72 | |||
73 | const ratingTypes = new Set(Object.values(VIDEO_RATE_TYPES)) | ||
74 | function isVideoRatingTypeValid (value: string) { | ||
75 | return value === 'none' || ratingTypes.has(value as VideoRateType) | ||
76 | } | ||
77 | |||
78 | function isVideoFileExtnameValid (value: string) { | ||
79 | return exists(value) && (value === VIDEO_LIVE.EXTENSION || MIMETYPES.VIDEO.EXT_MIMETYPE[value] !== undefined) | ||
80 | } | ||
81 | |||
82 | function isVideoFileMimeTypeValid (files: UploadFilesForCheck, field = 'videofile') { | ||
83 | return isFileValid({ | ||
84 | files, | ||
85 | mimeTypeRegex: MIMETYPES.VIDEO.MIMETYPES_REGEX, | ||
86 | field, | ||
87 | maxSize: null | ||
88 | }) | ||
89 | } | ||
90 | |||
91 | const videoImageTypes = CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME | ||
92 | .map(v => v.replace('.', '')) | ||
93 | .join('|') | ||
94 | const videoImageTypesRegex = `image/(${videoImageTypes})` | ||
95 | |||
96 | function isVideoImageValid (files: UploadFilesForCheck, field: string, optional = true) { | ||
97 | return isFileValid({ | ||
98 | files, | ||
99 | mimeTypeRegex: videoImageTypesRegex, | ||
100 | field, | ||
101 | maxSize: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max, | ||
102 | optional | ||
103 | }) | ||
104 | } | ||
105 | |||
106 | function isVideoPrivacyValid (value: number) { | ||
107 | return VIDEO_PRIVACIES[value] !== undefined | ||
108 | } | ||
109 | |||
110 | function isVideoReplayPrivacyValid (value: number) { | ||
111 | return VIDEO_PRIVACIES[value] !== undefined && value !== VideoPrivacy.PASSWORD_PROTECTED | ||
112 | } | ||
113 | |||
114 | function isScheduleVideoUpdatePrivacyValid (value: number) { | ||
115 | return value === VideoPrivacy.UNLISTED || value === VideoPrivacy.PUBLIC || value === VideoPrivacy.INTERNAL | ||
116 | } | ||
117 | |||
118 | function isVideoOriginallyPublishedAtValid (value: string | null) { | ||
119 | return value === null || isDateValid(value) | ||
120 | } | ||
121 | |||
122 | function isVideoFileInfoHashValid (value: string | null | undefined) { | ||
123 | return exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.INFO_HASH) | ||
124 | } | ||
125 | |||
126 | function isVideoFileResolutionValid (value: string) { | ||
127 | return exists(value) && validator.isInt(value + '') | ||
128 | } | ||
129 | |||
130 | function isVideoFPSResolutionValid (value: string) { | ||
131 | return value === null || validator.isInt(value + '') | ||
132 | } | ||
133 | |||
134 | function isVideoFileSizeValid (value: string) { | ||
135 | return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.FILE_SIZE) | ||
136 | } | ||
137 | |||
138 | function isVideoMagnetUriValid (value: string) { | ||
139 | if (!exists(value)) return false | ||
140 | |||
141 | const parsed = magnetUriDecode(value) | ||
142 | return parsed && isVideoFileInfoHashValid(parsed.infoHash) | ||
143 | } | ||
144 | |||
145 | function isPasswordValid (password: string) { | ||
146 | return password.length >= CONSTRAINTS_FIELDS.VIDEO_PASSWORD.LENGTH.min && | ||
147 | password.length < CONSTRAINTS_FIELDS.VIDEO_PASSWORD.LENGTH.max | ||
148 | } | ||
149 | |||
150 | function isValidPasswordProtectedPrivacy (req: Request, res: Response) { | ||
151 | const fail = (message: string) => { | ||
152 | res.fail({ | ||
153 | status: HttpStatusCode.BAD_REQUEST_400, | ||
154 | message | ||
155 | }) | ||
156 | return false | ||
157 | } | ||
158 | |||
159 | let privacy: VideoPrivacy | ||
160 | const video = getVideoWithAttributes(res) | ||
161 | |||
162 | if (exists(req.body?.privacy)) privacy = req.body.privacy | ||
163 | else if (exists(video?.privacy)) privacy = video.privacy | ||
164 | |||
165 | if (privacy !== VideoPrivacy.PASSWORD_PROTECTED) return true | ||
166 | |||
167 | if (!exists(req.body.videoPasswords) && !exists(req.body.passwords)) return fail('Video passwords are missing.') | ||
168 | |||
169 | const passwords = req.body.videoPasswords || req.body.passwords | ||
170 | |||
171 | if (passwords.length === 0) return fail('At least one video password is required.') | ||
172 | |||
173 | if (new Set(passwords).size !== passwords.length) return fail('Duplicate video passwords are not allowed.') | ||
174 | |||
175 | for (const password of passwords) { | ||
176 | if (typeof password !== 'string') { | ||
177 | return fail('Video password should be a string.') | ||
178 | } | ||
179 | |||
180 | if (!isPasswordValid(password)) { | ||
181 | return fail('Invalid video password. Password length should be at least 2 characters and no more than 100 characters.') | ||
182 | } | ||
183 | } | ||
184 | |||
185 | return true | ||
186 | } | ||
187 | |||
188 | // --------------------------------------------------------------------------- | ||
189 | |||
190 | export { | ||
191 | isVideoCategoryValid, | ||
192 | isVideoLicenceValid, | ||
193 | isVideoLanguageValid, | ||
194 | isVideoDescriptionValid, | ||
195 | isVideoFileInfoHashValid, | ||
196 | isVideoNameValid, | ||
197 | areVideoTagsValid, | ||
198 | isVideoFPSResolutionValid, | ||
199 | isScheduleVideoUpdatePrivacyValid, | ||
200 | isVideoOriginallyPublishedAtValid, | ||
201 | isVideoMagnetUriValid, | ||
202 | isVideoStateValid, | ||
203 | isVideoIncludeValid, | ||
204 | isVideoViewsValid, | ||
205 | isVideoRatingTypeValid, | ||
206 | isVideoFileExtnameValid, | ||
207 | isVideoFileMimeTypeValid, | ||
208 | isVideoDurationValid, | ||
209 | isVideoTagValid, | ||
210 | isVideoPrivacyValid, | ||
211 | isVideoReplayPrivacyValid, | ||
212 | isVideoFileResolutionValid, | ||
213 | isVideoFileSizeValid, | ||
214 | isVideoImageValid, | ||
215 | isVideoSupportValid, | ||
216 | isPasswordValid, | ||
217 | isValidPasswordProtectedPrivacy | ||
218 | } | ||
diff --git a/server/helpers/custom-validators/webfinger.ts b/server/helpers/custom-validators/webfinger.ts deleted file mode 100644 index dd914341e..000000000 --- a/server/helpers/custom-validators/webfinger.ts +++ /dev/null | |||
@@ -1,21 +0,0 @@ | |||
1 | import { REMOTE_SCHEME, WEBSERVER } from '../../initializers/constants' | ||
2 | import { sanitizeHost } from '../core-utils' | ||
3 | import { exists } from './misc' | ||
4 | |||
5 | function isWebfingerLocalResourceValid (value: string) { | ||
6 | if (!exists(value)) return false | ||
7 | if (value.startsWith('acct:') === false) return false | ||
8 | |||
9 | const actorWithHost = value.substr(5) | ||
10 | const actorParts = actorWithHost.split('@') | ||
11 | if (actorParts.length !== 2) return false | ||
12 | |||
13 | const host = actorParts[1] | ||
14 | return sanitizeHost(host, REMOTE_SCHEME.HTTP) === WEBSERVER.HOST | ||
15 | } | ||
16 | |||
17 | // --------------------------------------------------------------------------- | ||
18 | |||
19 | export { | ||
20 | isWebfingerLocalResourceValid | ||
21 | } | ||
diff --git a/server/helpers/database-utils.ts b/server/helpers/database-utils.ts deleted file mode 100644 index b6ba7fd75..000000000 --- a/server/helpers/database-utils.ts +++ /dev/null | |||
@@ -1,121 +0,0 @@ | |||
1 | import retry from 'async/retry' | ||
2 | import Bluebird from 'bluebird' | ||
3 | import { Transaction } from 'sequelize' | ||
4 | import { Model } from 'sequelize-typescript' | ||
5 | import { sequelizeTypescript } from '@server/initializers/database' | ||
6 | import { logger } from './logger' | ||
7 | |||
8 | function retryTransactionWrapper <T, A, B, C, D> ( | ||
9 | functionToRetry: (arg1: A, arg2: B, arg3: C, arg4: D) => Promise<T>, | ||
10 | arg1: A, | ||
11 | arg2: B, | ||
12 | arg3: C, | ||
13 | arg4: D, | ||
14 | ): Promise<T> | ||
15 | |||
16 | function retryTransactionWrapper <T, A, B, C> ( | ||
17 | functionToRetry: (arg1: A, arg2: B, arg3: C) => Promise<T>, | ||
18 | arg1: A, | ||
19 | arg2: B, | ||
20 | arg3: C | ||
21 | ): Promise<T> | ||
22 | |||
23 | function retryTransactionWrapper <T, A, B> ( | ||
24 | functionToRetry: (arg1: A, arg2: B) => Promise<T>, | ||
25 | arg1: A, | ||
26 | arg2: B | ||
27 | ): Promise<T> | ||
28 | |||
29 | function retryTransactionWrapper <T, A> ( | ||
30 | functionToRetry: (arg1: A) => Promise<T>, | ||
31 | arg1: A | ||
32 | ): Promise<T> | ||
33 | |||
34 | function retryTransactionWrapper <T> ( | ||
35 | functionToRetry: () => Promise<T> | Bluebird<T> | ||
36 | ): Promise<T> | ||
37 | |||
38 | function retryTransactionWrapper <T> ( | ||
39 | functionToRetry: (...args: any[]) => Promise<T>, | ||
40 | ...args: any[] | ||
41 | ): Promise<T> { | ||
42 | return transactionRetryer<T>(callback => { | ||
43 | functionToRetry.apply(null, args) | ||
44 | .then((result: T) => callback(null, result)) | ||
45 | .catch(err => callback(err)) | ||
46 | }) | ||
47 | .catch(err => { | ||
48 | logger.warn(`Cannot execute ${functionToRetry.name} with many retries.`, { err }) | ||
49 | throw err | ||
50 | }) | ||
51 | } | ||
52 | |||
53 | function transactionRetryer <T> (func: (err: any, data: T) => any) { | ||
54 | return new Promise<T>((res, rej) => { | ||
55 | retry( | ||
56 | { | ||
57 | times: 5, | ||
58 | |||
59 | errorFilter: err => { | ||
60 | const willRetry = (err.name === 'SequelizeDatabaseError') | ||
61 | logger.debug('Maybe retrying the transaction function.', { willRetry, err, tags: [ 'sql', 'retry' ] }) | ||
62 | return willRetry | ||
63 | } | ||
64 | }, | ||
65 | func, | ||
66 | (err, data) => err ? rej(err) : res(data) | ||
67 | ) | ||
68 | }) | ||
69 | } | ||
70 | |||
71 | function saveInTransactionWithRetries <T extends Pick<Model, 'save'>> (model: T) { | ||
72 | return retryTransactionWrapper(() => { | ||
73 | return sequelizeTypescript.transaction(async transaction => { | ||
74 | await model.save({ transaction }) | ||
75 | }) | ||
76 | }) | ||
77 | } | ||
78 | |||
79 | // --------------------------------------------------------------------------- | ||
80 | |||
81 | function resetSequelizeInstance <T> (instance: Model<T>) { | ||
82 | return instance.reload() | ||
83 | } | ||
84 | |||
85 | function filterNonExistingModels <T extends { hasSameUniqueKeysThan (other: T): boolean }> ( | ||
86 | fromDatabase: T[], | ||
87 | newModels: T[] | ||
88 | ) { | ||
89 | return fromDatabase.filter(f => !newModels.find(newModel => newModel.hasSameUniqueKeysThan(f))) | ||
90 | } | ||
91 | |||
92 | function deleteAllModels <T extends Pick<Model, 'destroy'>> (models: T[], transaction: Transaction) { | ||
93 | return Promise.all(models.map(f => f.destroy({ transaction }))) | ||
94 | } | ||
95 | |||
96 | // --------------------------------------------------------------------------- | ||
97 | |||
98 | function runInReadCommittedTransaction <T> (fn: (t: Transaction) => Promise<T>) { | ||
99 | const options = { isolationLevel: Transaction.ISOLATION_LEVELS.READ_COMMITTED } | ||
100 | |||
101 | return sequelizeTypescript.transaction(options, t => fn(t)) | ||
102 | } | ||
103 | |||
104 | function afterCommitIfTransaction (t: Transaction, fn: Function) { | ||
105 | if (t) return t.afterCommit(() => fn()) | ||
106 | |||
107 | return fn() | ||
108 | } | ||
109 | |||
110 | // --------------------------------------------------------------------------- | ||
111 | |||
112 | export { | ||
113 | resetSequelizeInstance, | ||
114 | retryTransactionWrapper, | ||
115 | transactionRetryer, | ||
116 | saveInTransactionWithRetries, | ||
117 | afterCommitIfTransaction, | ||
118 | filterNonExistingModels, | ||
119 | deleteAllModels, | ||
120 | runInReadCommittedTransaction | ||
121 | } | ||
diff --git a/server/helpers/debounce.ts b/server/helpers/debounce.ts deleted file mode 100644 index 77d99a894..000000000 --- a/server/helpers/debounce.ts +++ /dev/null | |||
@@ -1,16 +0,0 @@ | |||
1 | export function Debounce (config: { timeoutMS: number }) { | ||
2 | let timeoutRef: NodeJS.Timeout | ||
3 | |||
4 | return function (_target, _key, descriptor: PropertyDescriptor) { | ||
5 | const original = descriptor.value | ||
6 | |||
7 | descriptor.value = function (...args: any[]) { | ||
8 | clearTimeout(timeoutRef) | ||
9 | |||
10 | timeoutRef = setTimeout(() => { | ||
11 | original.apply(this, args) | ||
12 | |||
13 | }, config.timeoutMS) | ||
14 | } | ||
15 | } | ||
16 | } | ||
diff --git a/server/helpers/decache.ts b/server/helpers/decache.ts deleted file mode 100644 index 6be446ff6..000000000 --- a/server/helpers/decache.ts +++ /dev/null | |||
@@ -1,78 +0,0 @@ | |||
1 | // Thanks: https://github.com/dwyl/decache | ||
2 | // We reuse this file to also uncache plugin base path | ||
3 | |||
4 | import { extname } from 'path' | ||
5 | |||
6 | function decachePlugin (libraryPath: string) { | ||
7 | const moduleName = find(libraryPath) | ||
8 | |||
9 | if (!moduleName) return | ||
10 | |||
11 | searchCache(moduleName, function (mod) { | ||
12 | delete require.cache[mod.id] | ||
13 | |||
14 | removeCachedPath(mod.path) | ||
15 | }) | ||
16 | } | ||
17 | |||
18 | function decacheModule (name: string) { | ||
19 | const moduleName = find(name) | ||
20 | |||
21 | if (!moduleName) return | ||
22 | |||
23 | searchCache(moduleName, function (mod) { | ||
24 | delete require.cache[mod.id] | ||
25 | |||
26 | removeCachedPath(mod.path) | ||
27 | }) | ||
28 | } | ||
29 | |||
30 | // --------------------------------------------------------------------------- | ||
31 | |||
32 | export { | ||
33 | decacheModule, | ||
34 | decachePlugin | ||
35 | } | ||
36 | |||
37 | // --------------------------------------------------------------------------- | ||
38 | |||
39 | function find (moduleName: string) { | ||
40 | try { | ||
41 | return require.resolve(moduleName) | ||
42 | } catch { | ||
43 | return '' | ||
44 | } | ||
45 | } | ||
46 | |||
47 | function searchCache (moduleName: string, callback: (current: NodeModule) => void) { | ||
48 | const resolvedModule = require.resolve(moduleName) | ||
49 | let mod: NodeModule | ||
50 | const visited = {} | ||
51 | |||
52 | if (resolvedModule && ((mod = require.cache[resolvedModule]) !== undefined)) { | ||
53 | // Recursively go over the results | ||
54 | (function run (current) { | ||
55 | visited[current.id] = true | ||
56 | |||
57 | current.children.forEach(function (child) { | ||
58 | if (extname(child.filename) !== '.node' && !visited[child.id]) { | ||
59 | run(child) | ||
60 | } | ||
61 | }) | ||
62 | |||
63 | // Call the specified callback providing the | ||
64 | // found module | ||
65 | callback(current) | ||
66 | })(mod) | ||
67 | } | ||
68 | }; | ||
69 | |||
70 | function removeCachedPath (pluginPath: string) { | ||
71 | const pathCache = (module.constructor as any)._pathCache as { [ id: string ]: string[] } | ||
72 | |||
73 | Object.keys(pathCache).forEach(function (cacheKey) { | ||
74 | if (cacheKey.includes(pluginPath)) { | ||
75 | delete pathCache[cacheKey] | ||
76 | } | ||
77 | }) | ||
78 | } | ||
diff --git a/server/helpers/dns.ts b/server/helpers/dns.ts deleted file mode 100644 index da8b666c2..000000000 --- a/server/helpers/dns.ts +++ /dev/null | |||
@@ -1,29 +0,0 @@ | |||
1 | import { lookup } from 'dns' | ||
2 | import { parse as parseIP } from 'ipaddr.js' | ||
3 | |||
4 | function dnsLookupAll (hostname: string) { | ||
5 | return new Promise<string[]>((res, rej) => { | ||
6 | lookup(hostname, { family: 0, all: true }, (err, adresses) => { | ||
7 | if (err) return rej(err) | ||
8 | |||
9 | return res(adresses.map(a => a.address)) | ||
10 | }) | ||
11 | }) | ||
12 | } | ||
13 | |||
14 | async function isResolvingToUnicastOnly (hostname: string) { | ||
15 | const addresses = await dnsLookupAll(hostname) | ||
16 | |||
17 | for (const address of addresses) { | ||
18 | const parsed = parseIP(address) | ||
19 | |||
20 | if (parsed.range() !== 'unicast') return false | ||
21 | } | ||
22 | |||
23 | return true | ||
24 | } | ||
25 | |||
26 | export { | ||
27 | dnsLookupAll, | ||
28 | isResolvingToUnicastOnly | ||
29 | } | ||
diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts deleted file mode 100644 index 783097e55..000000000 --- a/server/helpers/express-utils.ts +++ /dev/null | |||
@@ -1,156 +0,0 @@ | |||
1 | import express, { RequestHandler } from 'express' | ||
2 | import multer, { diskStorage } from 'multer' | ||
3 | import { getLowercaseExtension } from '@shared/core-utils' | ||
4 | import { CONFIG } from '../initializers/config' | ||
5 | import { REMOTE_SCHEME } from '../initializers/constants' | ||
6 | import { isArray } from './custom-validators/misc' | ||
7 | import { logger } from './logger' | ||
8 | import { deleteFileAndCatch, generateRandomString } from './utils' | ||
9 | import { getExtFromMimetype } from './video' | ||
10 | |||
11 | function buildNSFWFilter (res?: express.Response, paramNSFW?: string) { | ||
12 | if (paramNSFW === 'true') return true | ||
13 | if (paramNSFW === 'false') return false | ||
14 | if (paramNSFW === 'both') return undefined | ||
15 | |||
16 | if (res?.locals.oauth) { | ||
17 | const user = res.locals.oauth.token.User | ||
18 | |||
19 | // User does not want NSFW videos | ||
20 | if (user.nsfwPolicy === 'do_not_list') return false | ||
21 | |||
22 | // Both | ||
23 | return undefined | ||
24 | } | ||
25 | |||
26 | if (CONFIG.INSTANCE.DEFAULT_NSFW_POLICY === 'do_not_list') return false | ||
27 | |||
28 | // Display all | ||
29 | return null | ||
30 | } | ||
31 | |||
32 | function cleanUpReqFiles (req: express.Request) { | ||
33 | const filesObject = req.files | ||
34 | if (!filesObject) return | ||
35 | |||
36 | if (isArray(filesObject)) { | ||
37 | filesObject.forEach(f => deleteFileAndCatch(f.path)) | ||
38 | return | ||
39 | } | ||
40 | |||
41 | for (const key of Object.keys(filesObject)) { | ||
42 | const files = filesObject[key] | ||
43 | |||
44 | files.forEach(f => deleteFileAndCatch(f.path)) | ||
45 | } | ||
46 | } | ||
47 | |||
48 | function getHostWithPort (host: string) { | ||
49 | const splitted = host.split(':') | ||
50 | |||
51 | // The port was not specified | ||
52 | if (splitted.length === 1) { | ||
53 | if (REMOTE_SCHEME.HTTP === 'https') return host + ':443' | ||
54 | |||
55 | return host + ':80' | ||
56 | } | ||
57 | |||
58 | return host | ||
59 | } | ||
60 | |||
61 | function createReqFiles ( | ||
62 | fieldNames: string[], | ||
63 | mimeTypes: { [id: string]: string | string[] }, | ||
64 | destination = CONFIG.STORAGE.TMP_DIR | ||
65 | ): RequestHandler { | ||
66 | const storage = diskStorage({ | ||
67 | destination: (req, file, cb) => { | ||
68 | cb(null, destination) | ||
69 | }, | ||
70 | |||
71 | filename: (req, file, cb) => { | ||
72 | return generateReqFilename(file, mimeTypes, cb) | ||
73 | } | ||
74 | }) | ||
75 | |||
76 | const fields: { name: string, maxCount: number }[] = [] | ||
77 | for (const fieldName of fieldNames) { | ||
78 | fields.push({ | ||
79 | name: fieldName, | ||
80 | maxCount: 1 | ||
81 | }) | ||
82 | } | ||
83 | |||
84 | return multer({ storage }).fields(fields) | ||
85 | } | ||
86 | |||
87 | function createAnyReqFiles ( | ||
88 | mimeTypes: { [id: string]: string | string[] }, | ||
89 | fileFilter: (req: express.Request, file: Express.Multer.File, cb: (err: Error, result: boolean) => void) => void | ||
90 | ): RequestHandler { | ||
91 | const storage = diskStorage({ | ||
92 | destination: (req, file, cb) => { | ||
93 | cb(null, CONFIG.STORAGE.TMP_DIR) | ||
94 | }, | ||
95 | |||
96 | filename: (req, file, cb) => { | ||
97 | return generateReqFilename(file, mimeTypes, cb) | ||
98 | } | ||
99 | }) | ||
100 | |||
101 | return multer({ storage, fileFilter }).any() | ||
102 | } | ||
103 | |||
104 | function isUserAbleToSearchRemoteURI (res: express.Response) { | ||
105 | const user = res.locals.oauth ? res.locals.oauth.token.User : undefined | ||
106 | |||
107 | return CONFIG.SEARCH.REMOTE_URI.ANONYMOUS === true || | ||
108 | (CONFIG.SEARCH.REMOTE_URI.USERS === true && user !== undefined) | ||
109 | } | ||
110 | |||
111 | function getCountVideos (req: express.Request) { | ||
112 | return req.query.skipCount !== true | ||
113 | } | ||
114 | |||
115 | // --------------------------------------------------------------------------- | ||
116 | |||
117 | export { | ||
118 | buildNSFWFilter, | ||
119 | getHostWithPort, | ||
120 | createAnyReqFiles, | ||
121 | isUserAbleToSearchRemoteURI, | ||
122 | createReqFiles, | ||
123 | cleanUpReqFiles, | ||
124 | getCountVideos | ||
125 | } | ||
126 | |||
127 | // --------------------------------------------------------------------------- | ||
128 | |||
129 | async function generateReqFilename ( | ||
130 | file: Express.Multer.File, | ||
131 | mimeTypes: { [id: string]: string | string[] }, | ||
132 | cb: (err: Error, name: string) => void | ||
133 | ) { | ||
134 | let extension: string | ||
135 | const fileExtension = getLowercaseExtension(file.originalname) | ||
136 | const extensionFromMimetype = getExtFromMimetype(mimeTypes, file.mimetype) | ||
137 | |||
138 | // Take the file extension if we don't understand the mime type | ||
139 | if (!extensionFromMimetype) { | ||
140 | extension = fileExtension | ||
141 | } else { | ||
142 | // Take the first available extension for this mimetype | ||
143 | extension = extensionFromMimetype | ||
144 | } | ||
145 | |||
146 | let randomString = '' | ||
147 | |||
148 | try { | ||
149 | randomString = await generateRandomString(16) | ||
150 | } catch (err) { | ||
151 | logger.error('Cannot generate random string for file name.', { err }) | ||
152 | randomString = 'fake-random-string' | ||
153 | } | ||
154 | |||
155 | cb(null, randomString + extension) | ||
156 | } | ||
diff --git a/server/helpers/ffmpeg/codecs.ts b/server/helpers/ffmpeg/codecs.ts deleted file mode 100644 index 3bd7db396..000000000 --- a/server/helpers/ffmpeg/codecs.ts +++ /dev/null | |||
@@ -1,64 +0,0 @@ | |||
1 | import { FfprobeData } from 'fluent-ffmpeg' | ||
2 | import { getAudioStream, getVideoStream } from '@shared/ffmpeg' | ||
3 | import { logger } from '../logger' | ||
4 | import { forceNumber } from '@shared/core-utils' | ||
5 | |||
6 | export async function getVideoStreamCodec (path: string) { | ||
7 | const videoStream = await getVideoStream(path) | ||
8 | if (!videoStream) return '' | ||
9 | |||
10 | const videoCodec = videoStream.codec_tag_string | ||
11 | |||
12 | if (videoCodec === 'vp09') return 'vp09.00.50.08' | ||
13 | if (videoCodec === 'hev1') return 'hev1.1.6.L93.B0' | ||
14 | |||
15 | const baseProfileMatrix = { | ||
16 | avc1: { | ||
17 | High: '6400', | ||
18 | Main: '4D40', | ||
19 | Baseline: '42E0' | ||
20 | }, | ||
21 | av01: { | ||
22 | High: '1', | ||
23 | Main: '0', | ||
24 | Professional: '2' | ||
25 | } | ||
26 | } | ||
27 | |||
28 | let baseProfile = baseProfileMatrix[videoCodec][videoStream.profile] | ||
29 | if (!baseProfile) { | ||
30 | logger.warn('Cannot get video profile codec of %s.', path, { videoStream }) | ||
31 | baseProfile = baseProfileMatrix[videoCodec]['High'] // Fallback | ||
32 | } | ||
33 | |||
34 | if (videoCodec === 'av01') { | ||
35 | let level = videoStream.level.toString() | ||
36 | if (level.length === 1) level = `0${level}` | ||
37 | |||
38 | // Guess the tier indicator and bit depth | ||
39 | return `${videoCodec}.${baseProfile}.${level}M.08` | ||
40 | } | ||
41 | |||
42 | let level = forceNumber(videoStream.level).toString(16) | ||
43 | if (level.length === 1) level = `0${level}` | ||
44 | |||
45 | // Default, h264 codec | ||
46 | return `${videoCodec}.${baseProfile}${level}` | ||
47 | } | ||
48 | |||
49 | export async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) { | ||
50 | const { audioStream } = await getAudioStream(path, existingProbe) | ||
51 | |||
52 | if (!audioStream) return '' | ||
53 | |||
54 | const audioCodecName = audioStream.codec_name | ||
55 | |||
56 | if (audioCodecName === 'opus') return 'opus' | ||
57 | if (audioCodecName === 'vorbis') return 'vorbis' | ||
58 | if (audioCodecName === 'aac') return 'mp4a.40.2' | ||
59 | if (audioCodecName === 'mp3') return 'mp4a.40.34' | ||
60 | |||
61 | logger.warn('Cannot get audio codec of %s.', path, { audioStream }) | ||
62 | |||
63 | return 'mp4a.40.2' // Fallback | ||
64 | } | ||
diff --git a/server/helpers/ffmpeg/ffmpeg-image.ts b/server/helpers/ffmpeg/ffmpeg-image.ts deleted file mode 100644 index 0bb0ff2c0..000000000 --- a/server/helpers/ffmpeg/ffmpeg-image.ts +++ /dev/null | |||
@@ -1,14 +0,0 @@ | |||
1 | import { FFmpegImage } from '@shared/ffmpeg' | ||
2 | import { getFFmpegCommandWrapperOptions } from './ffmpeg-options' | ||
3 | |||
4 | export function processGIF (options: Parameters<FFmpegImage['processGIF']>[0]) { | ||
5 | return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).processGIF(options) | ||
6 | } | ||
7 | |||
8 | export function generateThumbnailFromVideo (options: Parameters<FFmpegImage['generateThumbnailFromVideo']>[0]) { | ||
9 | return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).generateThumbnailFromVideo(options) | ||
10 | } | ||
11 | |||
12 | export function convertWebPToJPG (options: Parameters<FFmpegImage['convertWebPToJPG']>[0]) { | ||
13 | return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).convertWebPToJPG(options) | ||
14 | } | ||
diff --git a/server/helpers/ffmpeg/ffmpeg-options.ts b/server/helpers/ffmpeg/ffmpeg-options.ts deleted file mode 100644 index 64d7c4179..000000000 --- a/server/helpers/ffmpeg/ffmpeg-options.ts +++ /dev/null | |||
@@ -1,45 +0,0 @@ | |||
1 | import { logger } from '@server/helpers/logger' | ||
2 | import { CONFIG } from '@server/initializers/config' | ||
3 | import { FFMPEG_NICE } from '@server/initializers/constants' | ||
4 | import { FFmpegCommandWrapperOptions } from '@shared/ffmpeg' | ||
5 | import { AvailableEncoders } from '@shared/models' | ||
6 | |||
7 | type CommandType = 'live' | 'vod' | 'thumbnail' | ||
8 | |||
9 | export function getFFmpegCommandWrapperOptions (type: CommandType, availableEncoders?: AvailableEncoders): FFmpegCommandWrapperOptions { | ||
10 | return { | ||
11 | availableEncoders, | ||
12 | profile: getProfile(type), | ||
13 | |||
14 | niceness: FFMPEG_NICE[type.toUpperCase()], | ||
15 | tmpDirectory: CONFIG.STORAGE.TMP_DIR, | ||
16 | threads: getThreads(type), | ||
17 | |||
18 | logger: { | ||
19 | debug: logger.debug.bind(logger), | ||
20 | info: logger.info.bind(logger), | ||
21 | warn: logger.warn.bind(logger), | ||
22 | error: logger.error.bind(logger) | ||
23 | }, | ||
24 | lTags: { tags: [ 'ffmpeg' ] } | ||
25 | } | ||
26 | } | ||
27 | |||
28 | // --------------------------------------------------------------------------- | ||
29 | // Private | ||
30 | // --------------------------------------------------------------------------- | ||
31 | |||
32 | function getThreads (type: CommandType) { | ||
33 | if (type === 'live') return CONFIG.LIVE.TRANSCODING.THREADS | ||
34 | if (type === 'vod') return CONFIG.TRANSCODING.THREADS | ||
35 | |||
36 | // Auto | ||
37 | return 0 | ||
38 | } | ||
39 | |||
40 | function getProfile (type: CommandType) { | ||
41 | if (type === 'live') return CONFIG.LIVE.TRANSCODING.PROFILE | ||
42 | if (type === 'vod') return CONFIG.TRANSCODING.PROFILE | ||
43 | |||
44 | return undefined | ||
45 | } | ||
diff --git a/server/helpers/ffmpeg/framerate.ts b/server/helpers/ffmpeg/framerate.ts deleted file mode 100644 index 18cb0e0e2..000000000 --- a/server/helpers/ffmpeg/framerate.ts +++ /dev/null | |||
@@ -1,44 +0,0 @@ | |||
1 | import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' | ||
2 | import { VideoResolution } from '@shared/models' | ||
3 | |||
4 | export function computeOutputFPS (options: { | ||
5 | inputFPS: number | ||
6 | resolution: VideoResolution | ||
7 | }) { | ||
8 | const { resolution } = options | ||
9 | |||
10 | let fps = options.inputFPS | ||
11 | |||
12 | if ( | ||
13 | // On small/medium resolutions, limit FPS | ||
14 | resolution !== undefined && | ||
15 | resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN && | ||
16 | fps > VIDEO_TRANSCODING_FPS.AVERAGE | ||
17 | ) { | ||
18 | // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value | ||
19 | fps = getClosestFramerateStandard({ fps, type: 'STANDARD' }) | ||
20 | } | ||
21 | |||
22 | // Hard FPS limits | ||
23 | if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard({ fps, type: 'HD_STANDARD' }) | ||
24 | |||
25 | if (fps < VIDEO_TRANSCODING_FPS.MIN) { | ||
26 | throw new Error(`Cannot compute FPS because ${fps} is lower than our minimum value ${VIDEO_TRANSCODING_FPS.MIN}`) | ||
27 | } | ||
28 | |||
29 | return fps | ||
30 | } | ||
31 | |||
32 | // --------------------------------------------------------------------------- | ||
33 | // Private | ||
34 | // --------------------------------------------------------------------------- | ||
35 | |||
36 | function getClosestFramerateStandard (options: { | ||
37 | fps: number | ||
38 | type: 'HD_STANDARD' | 'STANDARD' | ||
39 | }) { | ||
40 | const { fps, type } = options | ||
41 | |||
42 | return VIDEO_TRANSCODING_FPS[type].slice(0) | ||
43 | .sort((a, b) => fps % a - fps % b)[0] | ||
44 | } | ||
diff --git a/server/helpers/ffmpeg/index.ts b/server/helpers/ffmpeg/index.ts deleted file mode 100644 index bf1c73fb6..000000000 --- a/server/helpers/ffmpeg/index.ts +++ /dev/null | |||
@@ -1,4 +0,0 @@ | |||
1 | export * from './codecs' | ||
2 | export * from './ffmpeg-image' | ||
3 | export * from './ffmpeg-options' | ||
4 | export * from './framerate' | ||
diff --git a/server/helpers/geo-ip.ts b/server/helpers/geo-ip.ts deleted file mode 100644 index 9e44d660f..000000000 --- a/server/helpers/geo-ip.ts +++ /dev/null | |||
@@ -1,78 +0,0 @@ | |||
1 | import { pathExists, writeFile } from 'fs-extra' | ||
2 | import maxmind, { CountryResponse, Reader } from 'maxmind' | ||
3 | import { join } from 'path' | ||
4 | import { CONFIG } from '@server/initializers/config' | ||
5 | import { logger, loggerTagsFactory } from './logger' | ||
6 | import { isBinaryResponse, peertubeGot } from './requests' | ||
7 | |||
8 | const lTags = loggerTagsFactory('geo-ip') | ||
9 | |||
10 | const mmbdFilename = 'dbip-country-lite-latest.mmdb' | ||
11 | const mmdbPath = join(CONFIG.STORAGE.BIN_DIR, mmbdFilename) | ||
12 | |||
13 | export class GeoIP { | ||
14 | private static instance: GeoIP | ||
15 | |||
16 | private reader: Reader<CountryResponse> | ||
17 | |||
18 | private constructor () { | ||
19 | } | ||
20 | |||
21 | async safeCountryISOLookup (ip: string): Promise<string> { | ||
22 | if (CONFIG.GEO_IP.ENABLED === false) return null | ||
23 | |||
24 | await this.initReaderIfNeeded() | ||
25 | |||
26 | try { | ||
27 | const result = this.reader.get(ip) | ||
28 | if (!result) return null | ||
29 | |||
30 | return result.country.iso_code | ||
31 | } catch (err) { | ||
32 | logger.error('Cannot get country from IP.', { err }) | ||
33 | |||
34 | return null | ||
35 | } | ||
36 | } | ||
37 | |||
38 | async updateDatabase () { | ||
39 | if (CONFIG.GEO_IP.ENABLED === false) return | ||
40 | |||
41 | const url = CONFIG.GEO_IP.COUNTRY.DATABASE_URL | ||
42 | |||
43 | logger.info('Updating GeoIP database from %s.', url, lTags()) | ||
44 | |||
45 | const gotOptions = { context: { bodyKBLimit: 200_000 }, responseType: 'buffer' as 'buffer' } | ||
46 | |||
47 | try { | ||
48 | const gotResult = await peertubeGot(url, gotOptions) | ||
49 | |||
50 | if (!isBinaryResponse(gotResult)) { | ||
51 | throw new Error('Not a binary response') | ||
52 | } | ||
53 | |||
54 | await writeFile(mmdbPath, gotResult.body) | ||
55 | |||
56 | // Reinit reader | ||
57 | this.reader = undefined | ||
58 | |||
59 | logger.info('GeoIP database updated %s.', mmdbPath, lTags()) | ||
60 | } catch (err) { | ||
61 | logger.error('Cannot update GeoIP database from %s.', url, { err, ...lTags() }) | ||
62 | } | ||
63 | } | ||
64 | |||
65 | private async initReaderIfNeeded () { | ||
66 | if (!this.reader) { | ||
67 | if (!await pathExists(mmdbPath)) { | ||
68 | await this.updateDatabase() | ||
69 | } | ||
70 | |||
71 | this.reader = await maxmind.open(mmdbPath) | ||
72 | } | ||
73 | } | ||
74 | |||
75 | static get Instance () { | ||
76 | return this.instance || (this.instance = new this()) | ||
77 | } | ||
78 | } | ||
diff --git a/server/helpers/image-utils.ts b/server/helpers/image-utils.ts deleted file mode 100644 index 2a8bb6e6e..000000000 --- a/server/helpers/image-utils.ts +++ /dev/null | |||
@@ -1,179 +0,0 @@ | |||
1 | import { copy, readFile, remove, rename } from 'fs-extra' | ||
2 | import Jimp, { read as jimpRead } from 'jimp' | ||
3 | import { join } from 'path' | ||
4 | import { ColorActionName } from '@jimp/plugin-color' | ||
5 | import { getLowercaseExtension } from '@shared/core-utils' | ||
6 | import { buildUUID } from '@shared/extra-utils' | ||
7 | import { convertWebPToJPG, generateThumbnailFromVideo, processGIF } from './ffmpeg' | ||
8 | import { logger, loggerTagsFactory } from './logger' | ||
9 | |||
10 | const lTags = loggerTagsFactory('image-utils') | ||
11 | |||
12 | function generateImageFilename (extension = '.jpg') { | ||
13 | return buildUUID() + extension | ||
14 | } | ||
15 | |||
16 | async function processImage (options: { | ||
17 | path: string | ||
18 | destination: string | ||
19 | newSize: { width: number, height: number } | ||
20 | keepOriginal?: boolean // default false | ||
21 | }) { | ||
22 | const { path, destination, newSize, keepOriginal = false } = options | ||
23 | |||
24 | const extension = getLowercaseExtension(path) | ||
25 | |||
26 | if (path === destination) { | ||
27 | throw new Error('Jimp/FFmpeg needs an input path different that the output path.') | ||
28 | } | ||
29 | |||
30 | logger.debug('Processing image %s to %s.', path, destination) | ||
31 | |||
32 | // Use FFmpeg to process GIF | ||
33 | if (extension === '.gif') { | ||
34 | await processGIF({ path, destination, newSize }) | ||
35 | } else { | ||
36 | await jimpProcessor(path, destination, newSize, extension) | ||
37 | } | ||
38 | |||
39 | if (keepOriginal !== true) await remove(path) | ||
40 | } | ||
41 | |||
42 | async function generateImageFromVideoFile (options: { | ||
43 | fromPath: string | ||
44 | folder: string | ||
45 | imageName: string | ||
46 | size: { width: number, height: number } | ||
47 | }) { | ||
48 | const { fromPath, folder, imageName, size } = options | ||
49 | |||
50 | const pendingImageName = 'pending-' + imageName | ||
51 | const pendingImagePath = join(folder, pendingImageName) | ||
52 | |||
53 | try { | ||
54 | await generateThumbnailFromVideo({ fromPath, output: pendingImagePath }) | ||
55 | |||
56 | const destination = join(folder, imageName) | ||
57 | await processImage({ path: pendingImagePath, destination, newSize: size }) | ||
58 | } catch (err) { | ||
59 | logger.error('Cannot generate image from video %s.', fromPath, { err, ...lTags() }) | ||
60 | |||
61 | try { | ||
62 | await remove(pendingImagePath) | ||
63 | } catch (err) { | ||
64 | logger.debug('Cannot remove pending image path after generation error.', { err, ...lTags() }) | ||
65 | } | ||
66 | |||
67 | throw err | ||
68 | } | ||
69 | } | ||
70 | |||
71 | async function getImageSize (path: string) { | ||
72 | const inputBuffer = await readFile(path) | ||
73 | |||
74 | const image = await jimpRead(inputBuffer) | ||
75 | |||
76 | return { | ||
77 | width: image.getWidth(), | ||
78 | height: image.getHeight() | ||
79 | } | ||
80 | } | ||
81 | |||
82 | // --------------------------------------------------------------------------- | ||
83 | |||
84 | export { | ||
85 | generateImageFilename, | ||
86 | generateImageFromVideoFile, | ||
87 | |||
88 | processImage, | ||
89 | |||
90 | getImageSize | ||
91 | } | ||
92 | |||
93 | // --------------------------------------------------------------------------- | ||
94 | |||
95 | async function jimpProcessor (path: string, destination: string, newSize: { width: number, height: number }, inputExt: string) { | ||
96 | let sourceImage: Jimp | ||
97 | const inputBuffer = await readFile(path) | ||
98 | |||
99 | try { | ||
100 | sourceImage = await jimpRead(inputBuffer) | ||
101 | } catch (err) { | ||
102 | logger.debug('Cannot read %s with jimp. Try to convert the image using ffmpeg first.', path, { err }) | ||
103 | |||
104 | const newName = path + '.jpg' | ||
105 | await convertWebPToJPG({ path, destination: newName }) | ||
106 | await rename(newName, path) | ||
107 | |||
108 | sourceImage = await jimpRead(path) | ||
109 | } | ||
110 | |||
111 | await remove(destination) | ||
112 | |||
113 | // Optimization if the source file has the appropriate size | ||
114 | const outputExt = getLowercaseExtension(destination) | ||
115 | if (skipProcessing({ sourceImage, newSize, imageBytes: inputBuffer.byteLength, inputExt, outputExt })) { | ||
116 | return copy(path, destination) | ||
117 | } | ||
118 | |||
119 | await autoResize({ sourceImage, newSize, destination }) | ||
120 | } | ||
121 | |||
122 | async function autoResize (options: { | ||
123 | sourceImage: Jimp | ||
124 | newSize: { width: number, height: number } | ||
125 | destination: string | ||
126 | }) { | ||
127 | const { sourceImage, newSize, destination } = options | ||
128 | |||
129 | // Portrait mode targeting a landscape, apply some effect on the image | ||
130 | const sourceIsPortrait = sourceImage.getWidth() < sourceImage.getHeight() | ||
131 | const destIsPortraitOrSquare = newSize.width <= newSize.height | ||
132 | |||
133 | removeExif(sourceImage) | ||
134 | |||
135 | if (sourceIsPortrait && !destIsPortraitOrSquare) { | ||
136 | const baseImage = sourceImage.cloneQuiet().cover(newSize.width, newSize.height) | ||
137 | .color([ { apply: ColorActionName.SHADE, params: [ 50 ] } ]) | ||
138 | |||
139 | const topImage = sourceImage.cloneQuiet().contain(newSize.width, newSize.height) | ||
140 | |||
141 | return write(baseImage.blit(topImage, 0, 0), destination) | ||
142 | } | ||
143 | |||
144 | return write(sourceImage.cover(newSize.width, newSize.height), destination) | ||
145 | } | ||
146 | |||
147 | function write (image: Jimp, destination: string) { | ||
148 | return image.quality(80).writeAsync(destination) | ||
149 | } | ||
150 | |||
151 | function skipProcessing (options: { | ||
152 | sourceImage: Jimp | ||
153 | newSize: { width: number, height: number } | ||
154 | imageBytes: number | ||
155 | inputExt: string | ||
156 | outputExt: string | ||
157 | }) { | ||
158 | const { sourceImage, newSize, imageBytes, inputExt, outputExt } = options | ||
159 | const { width, height } = newSize | ||
160 | |||
161 | if (hasExif(sourceImage)) return false | ||
162 | if (sourceImage.getWidth() > width || sourceImage.getHeight() > height) return false | ||
163 | if (inputExt !== outputExt) return false | ||
164 | |||
165 | const kB = 1000 | ||
166 | |||
167 | if (height >= 1000) return imageBytes <= 200 * kB | ||
168 | if (height >= 500) return imageBytes <= 100 * kB | ||
169 | |||
170 | return imageBytes <= 15 * kB | ||
171 | } | ||
172 | |||
173 | function hasExif (image: Jimp) { | ||
174 | return !!(image.bitmap as any).exifBuffer | ||
175 | } | ||
176 | |||
177 | function removeExif (image: Jimp) { | ||
178 | (image.bitmap as any).exifBuffer = null | ||
179 | } | ||
diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts deleted file mode 100644 index 6649db40f..000000000 --- a/server/helpers/logger.ts +++ /dev/null | |||
@@ -1,208 +0,0 @@ | |||
1 | import { stat } from 'fs-extra' | ||
2 | import { join } from 'path' | ||
3 | import { format as sqlFormat } from 'sql-formatter' | ||
4 | import { createLogger, format, transports } from 'winston' | ||
5 | import { FileTransportOptions } from 'winston/lib/winston/transports' | ||
6 | import { context } from '@opentelemetry/api' | ||
7 | import { getSpanContext } from '@opentelemetry/api/build/src/trace/context-utils' | ||
8 | import { omit } from '@shared/core-utils' | ||
9 | import { CONFIG } from '../initializers/config' | ||
10 | import { LOG_FILENAME } from '../initializers/constants' | ||
11 | |||
12 | const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT | ||
13 | |||
14 | const consoleLoggerFormat = format.printf(info => { | ||
15 | let additionalInfos = JSON.stringify(getAdditionalInfo(info), removeCyclicValues(), 2) | ||
16 | |||
17 | if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = '' | ||
18 | else additionalInfos = ' ' + additionalInfos | ||
19 | |||
20 | if (info.sql) { | ||
21 | if (CONFIG.LOG.PRETTIFY_SQL) { | ||
22 | additionalInfos += '\n' + sqlFormat(info.sql, { | ||
23 | language: 'sql', | ||
24 | tabWidth: 2 | ||
25 | }) | ||
26 | } else { | ||
27 | additionalInfos += ' - ' + info.sql | ||
28 | } | ||
29 | } | ||
30 | |||
31 | return `[${info.label}] ${info.timestamp} ${info.level}: ${info.message}${additionalInfos}` | ||
32 | }) | ||
33 | |||
34 | const jsonLoggerFormat = format.printf(info => { | ||
35 | return JSON.stringify(info, removeCyclicValues()) | ||
36 | }) | ||
37 | |||
38 | const timestampFormatter = format.timestamp({ | ||
39 | format: 'YYYY-MM-DD HH:mm:ss.SSS' | ||
40 | }) | ||
41 | const labelFormatter = (suffix?: string) => { | ||
42 | return format.label({ | ||
43 | label: suffix ? `${label} ${suffix}` : label | ||
44 | }) | ||
45 | } | ||
46 | |||
47 | const fileLoggerOptions: FileTransportOptions = { | ||
48 | filename: join(CONFIG.STORAGE.LOG_DIR, LOG_FILENAME), | ||
49 | handleExceptions: true, | ||
50 | format: format.combine( | ||
51 | format.timestamp(), | ||
52 | jsonLoggerFormat | ||
53 | ) | ||
54 | } | ||
55 | |||
56 | if (CONFIG.LOG.ROTATION.ENABLED) { | ||
57 | fileLoggerOptions.maxsize = CONFIG.LOG.ROTATION.MAX_FILE_SIZE | ||
58 | fileLoggerOptions.maxFiles = CONFIG.LOG.ROTATION.MAX_FILES | ||
59 | } | ||
60 | |||
61 | function buildLogger (labelSuffix?: string) { | ||
62 | return createLogger({ | ||
63 | level: CONFIG.LOG.LEVEL, | ||
64 | defaultMeta: { | ||
65 | get traceId () { return getSpanContext(context.active())?.traceId }, | ||
66 | get spanId () { return getSpanContext(context.active())?.spanId }, | ||
67 | get traceFlags () { return getSpanContext(context.active())?.traceFlags } | ||
68 | }, | ||
69 | format: format.combine( | ||
70 | labelFormatter(labelSuffix), | ||
71 | format.splat() | ||
72 | ), | ||
73 | transports: [ | ||
74 | new transports.File(fileLoggerOptions), | ||
75 | new transports.Console({ | ||
76 | handleExceptions: true, | ||
77 | format: format.combine( | ||
78 | timestampFormatter, | ||
79 | format.colorize(), | ||
80 | consoleLoggerFormat | ||
81 | ) | ||
82 | }) | ||
83 | ], | ||
84 | exitOnError: true | ||
85 | }) | ||
86 | } | ||
87 | |||
88 | const logger = buildLogger() | ||
89 | |||
90 | // --------------------------------------------------------------------------- | ||
91 | |||
92 | function bunyanLogFactory (level: string) { | ||
93 | return function (...params: any[]) { | ||
94 | let meta = null | ||
95 | let args = [].concat(params) | ||
96 | |||
97 | if (arguments[0] instanceof Error) { | ||
98 | meta = arguments[0].toString() | ||
99 | args = Array.prototype.slice.call(arguments, 1) | ||
100 | args.push(meta) | ||
101 | } else if (typeof (args[0]) !== 'string') { | ||
102 | meta = arguments[0] | ||
103 | args = Array.prototype.slice.call(arguments, 1) | ||
104 | args.push(meta) | ||
105 | } | ||
106 | |||
107 | logger[level].apply(logger, args) | ||
108 | } | ||
109 | } | ||
110 | |||
111 | const bunyanLogger = { | ||
112 | level: () => { }, | ||
113 | trace: bunyanLogFactory('debug'), | ||
114 | debug: bunyanLogFactory('debug'), | ||
115 | verbose: bunyanLogFactory('debug'), | ||
116 | info: bunyanLogFactory('info'), | ||
117 | warn: bunyanLogFactory('warn'), | ||
118 | error: bunyanLogFactory('error'), | ||
119 | fatal: bunyanLogFactory('error') | ||
120 | } | ||
121 | |||
122 | // --------------------------------------------------------------------------- | ||
123 | |||
124 | type LoggerTagsFn = (...tags: string[]) => { tags: string[] } | ||
125 | function loggerTagsFactory (...defaultTags: string[]): LoggerTagsFn { | ||
126 | return (...tags: string[]) => { | ||
127 | return { tags: defaultTags.concat(tags) } | ||
128 | } | ||
129 | } | ||
130 | |||
131 | // --------------------------------------------------------------------------- | ||
132 | |||
133 | async function mtimeSortFilesDesc (files: string[], basePath: string) { | ||
134 | const promises = [] | ||
135 | const out: { file: string, mtime: number }[] = [] | ||
136 | |||
137 | for (const file of files) { | ||
138 | const p = stat(basePath + '/' + file) | ||
139 | .then(stats => { | ||
140 | if (stats.isFile()) out.push({ file, mtime: stats.mtime.getTime() }) | ||
141 | }) | ||
142 | |||
143 | promises.push(p) | ||
144 | } | ||
145 | |||
146 | await Promise.all(promises) | ||
147 | |||
148 | out.sort((a, b) => b.mtime - a.mtime) | ||
149 | |||
150 | return out | ||
151 | } | ||
152 | |||
153 | // --------------------------------------------------------------------------- | ||
154 | |||
155 | export { | ||
156 | LoggerTagsFn, | ||
157 | |||
158 | buildLogger, | ||
159 | timestampFormatter, | ||
160 | labelFormatter, | ||
161 | consoleLoggerFormat, | ||
162 | jsonLoggerFormat, | ||
163 | mtimeSortFilesDesc, | ||
164 | logger, | ||
165 | loggerTagsFactory, | ||
166 | bunyanLogger | ||
167 | } | ||
168 | |||
169 | // --------------------------------------------------------------------------- | ||
170 | |||
171 | function removeCyclicValues () { | ||
172 | const seen = new WeakSet() | ||
173 | |||
174 | // Thanks: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Cyclic_object_value#Examples | ||
175 | return (key: string, value: any) => { | ||
176 | if (key === 'cert') return 'Replaced by the logger to avoid large log message' | ||
177 | |||
178 | if (typeof value === 'object' && value !== null) { | ||
179 | if (seen.has(value)) return | ||
180 | |||
181 | seen.add(value) | ||
182 | } | ||
183 | |||
184 | if (value instanceof Set) { | ||
185 | return Array.from(value) | ||
186 | } | ||
187 | |||
188 | if (value instanceof Map) { | ||
189 | return Array.from(value.entries()) | ||
190 | } | ||
191 | |||
192 | if (value instanceof Error) { | ||
193 | const error = {} | ||
194 | |||
195 | Object.getOwnPropertyNames(value).forEach(key => { error[key] = value[key] }) | ||
196 | |||
197 | return error | ||
198 | } | ||
199 | |||
200 | return value | ||
201 | } | ||
202 | } | ||
203 | |||
204 | function getAdditionalInfo (info: any) { | ||
205 | const toOmit = [ 'label', 'timestamp', 'level', 'message', 'sql', 'tags' ] | ||
206 | |||
207 | return omit(info, toOmit) | ||
208 | } | ||
diff --git a/server/helpers/markdown.ts b/server/helpers/markdown.ts deleted file mode 100644 index a20ac22d4..000000000 --- a/server/helpers/markdown.ts +++ /dev/null | |||
@@ -1,90 +0,0 @@ | |||
1 | import { getDefaultSanitizeOptions, getTextOnlySanitizeOptions, TEXT_WITH_HTML_RULES } from '@shared/core-utils' | ||
2 | |||
3 | const defaultSanitizeOptions = getDefaultSanitizeOptions() | ||
4 | const textOnlySanitizeOptions = getTextOnlySanitizeOptions() | ||
5 | |||
6 | const sanitizeHtml = require('sanitize-html') | ||
7 | const markdownItEmoji = require('markdown-it-emoji/light') | ||
8 | const MarkdownItClass = require('markdown-it') | ||
9 | |||
10 | const markdownItForSafeHtml = new MarkdownItClass('default', { linkify: true, breaks: true, html: true }) | ||
11 | .enable(TEXT_WITH_HTML_RULES) | ||
12 | .use(markdownItEmoji) | ||
13 | |||
14 | const markdownItForPlainText = new MarkdownItClass('default', { linkify: false, breaks: true, html: false }) | ||
15 | .use(markdownItEmoji) | ||
16 | .use(plainTextPlugin) | ||
17 | |||
18 | const toSafeHtml = (text: string) => { | ||
19 | if (!text) return '' | ||
20 | |||
21 | // Restore line feed | ||
22 | const textWithLineFeed = text.replace(/<br.?\/?>/g, '\r\n') | ||
23 | |||
24 | // Convert possible markdown (emojis, emphasis and lists) to html | ||
25 | const html = markdownItForSafeHtml.render(textWithLineFeed) | ||
26 | |||
27 | // Convert to safe Html | ||
28 | return sanitizeHtml(html, defaultSanitizeOptions) | ||
29 | } | ||
30 | |||
31 | const mdToOneLinePlainText = (text: string) => { | ||
32 | if (!text) return '' | ||
33 | |||
34 | markdownItForPlainText.render(text) | ||
35 | |||
36 | // Convert to safe Html | ||
37 | return sanitizeHtml(markdownItForPlainText.plainText, textOnlySanitizeOptions) | ||
38 | } | ||
39 | |||
40 | // --------------------------------------------------------------------------- | ||
41 | |||
42 | export { | ||
43 | toSafeHtml, | ||
44 | mdToOneLinePlainText | ||
45 | } | ||
46 | |||
47 | // --------------------------------------------------------------------------- | ||
48 | |||
49 | // Thanks: https://github.com/wavesheep/markdown-it-plain-text | ||
50 | function plainTextPlugin (markdownIt: any) { | ||
51 | function plainTextRule (state: any) { | ||
52 | const text = scan(state.tokens) | ||
53 | |||
54 | markdownIt.plainText = text | ||
55 | } | ||
56 | |||
57 | function scan (tokens: any[]) { | ||
58 | let lastSeparator = '' | ||
59 | let text = '' | ||
60 | |||
61 | function buildSeparator (token: any) { | ||
62 | if (token.type === 'list_item_close') { | ||
63 | lastSeparator = ', ' | ||
64 | } | ||
65 | |||
66 | if (token.tag === 'br' || token.type === 'paragraph_close') { | ||
67 | lastSeparator = ' ' | ||
68 | } | ||
69 | } | ||
70 | |||
71 | for (const token of tokens) { | ||
72 | buildSeparator(token) | ||
73 | |||
74 | if (token.type !== 'inline') continue | ||
75 | |||
76 | for (const child of token.children) { | ||
77 | buildSeparator(child) | ||
78 | |||
79 | if (!child.content) continue | ||
80 | |||
81 | text += lastSeparator + child.content | ||
82 | lastSeparator = '' | ||
83 | } | ||
84 | } | ||
85 | |||
86 | return text | ||
87 | } | ||
88 | |||
89 | markdownIt.core.ruler.push('plainText', plainTextRule) | ||
90 | } | ||
diff --git a/server/helpers/memoize.ts b/server/helpers/memoize.ts deleted file mode 100644 index aa20e7d73..000000000 --- a/server/helpers/memoize.ts +++ /dev/null | |||
@@ -1,12 +0,0 @@ | |||
1 | import memoizee from 'memoizee' | ||
2 | |||
3 | export function Memoize (config?: memoizee.Options<any>) { | ||
4 | return function (_target, _key, descriptor: PropertyDescriptor) { | ||
5 | const oldFunction = descriptor.value | ||
6 | const newFunction = memoizee(oldFunction, config) | ||
7 | |||
8 | descriptor.value = function () { | ||
9 | return newFunction.apply(this, arguments) | ||
10 | } | ||
11 | } | ||
12 | } | ||
diff --git a/server/helpers/otp.ts b/server/helpers/otp.ts deleted file mode 100644 index a32cc9621..000000000 --- a/server/helpers/otp.ts +++ /dev/null | |||
@@ -1,58 +0,0 @@ | |||
1 | import { Secret, TOTP } from 'otpauth' | ||
2 | import { CONFIG } from '@server/initializers/config' | ||
3 | import { WEBSERVER } from '@server/initializers/constants' | ||
4 | import { decrypt } from './peertube-crypto' | ||
5 | |||
6 | async function isOTPValid (options: { | ||
7 | encryptedSecret: string | ||
8 | token: string | ||
9 | }) { | ||
10 | const { token, encryptedSecret } = options | ||
11 | |||
12 | const secret = await decrypt(encryptedSecret, CONFIG.SECRETS.PEERTUBE) | ||
13 | |||
14 | const totp = new TOTP({ | ||
15 | ...baseOTPOptions(), | ||
16 | |||
17 | secret | ||
18 | }) | ||
19 | |||
20 | const delta = totp.validate({ | ||
21 | token, | ||
22 | window: 1 | ||
23 | }) | ||
24 | |||
25 | if (delta === null) return false | ||
26 | |||
27 | return true | ||
28 | } | ||
29 | |||
30 | function generateOTPSecret (email: string) { | ||
31 | const totp = new TOTP({ | ||
32 | ...baseOTPOptions(), | ||
33 | |||
34 | label: email, | ||
35 | secret: new Secret() | ||
36 | }) | ||
37 | |||
38 | return { | ||
39 | secret: totp.secret.base32, | ||
40 | uri: totp.toString() | ||
41 | } | ||
42 | } | ||
43 | |||
44 | export { | ||
45 | isOTPValid, | ||
46 | generateOTPSecret | ||
47 | } | ||
48 | |||
49 | // --------------------------------------------------------------------------- | ||
50 | |||
51 | function baseOTPOptions () { | ||
52 | return { | ||
53 | issuer: WEBSERVER.HOST, | ||
54 | algorithm: 'SHA1', | ||
55 | digits: 6, | ||
56 | period: 30 | ||
57 | } | ||
58 | } | ||
diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts deleted file mode 100644 index 95e78a904..000000000 --- a/server/helpers/peertube-crypto.ts +++ /dev/null | |||
@@ -1,208 +0,0 @@ | |||
1 | import { compare, genSalt, hash } from 'bcrypt' | ||
2 | import { createCipheriv, createDecipheriv, createSign, createVerify } from 'crypto' | ||
3 | import { Request } from 'express' | ||
4 | import { cloneDeep } from 'lodash' | ||
5 | import { promisify1, promisify2 } from '@shared/core-utils' | ||
6 | import { sha256 } from '@shared/extra-utils' | ||
7 | import { BCRYPT_SALT_SIZE, ENCRYPTION, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants' | ||
8 | import { MActor } from '../types/models' | ||
9 | import { generateRSAKeyPairPromise, randomBytesPromise, scryptPromise } from './core-utils' | ||
10 | import { jsonld } from './custom-jsonld-signature' | ||
11 | import { logger } from './logger' | ||
12 | |||
13 | const bcryptComparePromise = promisify2<any, string, boolean>(compare) | ||
14 | const bcryptGenSaltPromise = promisify1<number, string>(genSalt) | ||
15 | const bcryptHashPromise = promisify2<any, string | number, string>(hash) | ||
16 | |||
17 | const httpSignature = require('@peertube/http-signature') | ||
18 | |||
19 | function createPrivateAndPublicKeys () { | ||
20 | logger.info('Generating a RSA key...') | ||
21 | |||
22 | return generateRSAKeyPairPromise(PRIVATE_RSA_KEY_SIZE) | ||
23 | } | ||
24 | |||
25 | // --------------------------------------------------------------------------- | ||
26 | // User password checks | ||
27 | // --------------------------------------------------------------------------- | ||
28 | |||
29 | function comparePassword (plainPassword: string, hashPassword: string) { | ||
30 | if (!plainPassword) return Promise.resolve(false) | ||
31 | |||
32 | return bcryptComparePromise(plainPassword, hashPassword) | ||
33 | } | ||
34 | |||
35 | async function cryptPassword (password: string) { | ||
36 | const salt = await bcryptGenSaltPromise(BCRYPT_SALT_SIZE) | ||
37 | |||
38 | return bcryptHashPromise(password, salt) | ||
39 | } | ||
40 | |||
41 | // --------------------------------------------------------------------------- | ||
42 | // HTTP Signature | ||
43 | // --------------------------------------------------------------------------- | ||
44 | |||
45 | function isHTTPSignatureDigestValid (rawBody: Buffer, req: Request): boolean { | ||
46 | if (req.headers[HTTP_SIGNATURE.HEADER_NAME] && req.headers['digest']) { | ||
47 | return buildDigest(rawBody.toString()) === req.headers['digest'] | ||
48 | } | ||
49 | |||
50 | return true | ||
51 | } | ||
52 | |||
53 | function isHTTPSignatureVerified (httpSignatureParsed: any, actor: MActor): boolean { | ||
54 | return httpSignature.verifySignature(httpSignatureParsed, actor.publicKey) === true | ||
55 | } | ||
56 | |||
57 | function parseHTTPSignature (req: Request, clockSkew?: number) { | ||
58 | const requiredHeaders = req.method === 'POST' | ||
59 | ? [ '(request-target)', 'host', 'digest' ] | ||
60 | : [ '(request-target)', 'host' ] | ||
61 | |||
62 | const parsed = httpSignature.parse(req, { clockSkew, headers: requiredHeaders }) | ||
63 | |||
64 | const parsedHeaders = parsed.params.headers | ||
65 | if (!parsedHeaders.includes('date') && !parsedHeaders.includes('(created)')) { | ||
66 | throw new Error(`date or (created) must be included in signature`) | ||
67 | } | ||
68 | |||
69 | return parsed | ||
70 | } | ||
71 | |||
72 | // --------------------------------------------------------------------------- | ||
73 | // JSONLD | ||
74 | // --------------------------------------------------------------------------- | ||
75 | |||
76 | function isJsonLDSignatureVerified (fromActor: MActor, signedDocument: any): Promise<boolean> { | ||
77 | if (signedDocument.signature.type === 'RsaSignature2017') { | ||
78 | return isJsonLDRSA2017Verified(fromActor, signedDocument) | ||
79 | } | ||
80 | |||
81 | logger.warn('Unknown JSON LD signature %s.', signedDocument.signature.type, signedDocument) | ||
82 | |||
83 | return Promise.resolve(false) | ||
84 | } | ||
85 | |||
86 | // Backward compatibility with "other" implementations | ||
87 | async function isJsonLDRSA2017Verified (fromActor: MActor, signedDocument: any) { | ||
88 | const [ documentHash, optionsHash ] = await Promise.all([ | ||
89 | createDocWithoutSignatureHash(signedDocument), | ||
90 | createSignatureHash(signedDocument.signature) | ||
91 | ]) | ||
92 | |||
93 | const toVerify = optionsHash + documentHash | ||
94 | |||
95 | const verify = createVerify('RSA-SHA256') | ||
96 | verify.update(toVerify, 'utf8') | ||
97 | |||
98 | return verify.verify(fromActor.publicKey, signedDocument.signature.signatureValue, 'base64') | ||
99 | } | ||
100 | |||
101 | async function signJsonLDObject <T> (byActor: MActor, data: T) { | ||
102 | const signature = { | ||
103 | type: 'RsaSignature2017', | ||
104 | creator: byActor.url, | ||
105 | created: new Date().toISOString() | ||
106 | } | ||
107 | |||
108 | const [ documentHash, optionsHash ] = await Promise.all([ | ||
109 | createDocWithoutSignatureHash(data), | ||
110 | createSignatureHash(signature) | ||
111 | ]) | ||
112 | |||
113 | const toSign = optionsHash + documentHash | ||
114 | |||
115 | const sign = createSign('RSA-SHA256') | ||
116 | sign.update(toSign, 'utf8') | ||
117 | |||
118 | const signatureValue = sign.sign(byActor.privateKey, 'base64') | ||
119 | Object.assign(signature, { signatureValue }) | ||
120 | |||
121 | return Object.assign(data, { signature }) | ||
122 | } | ||
123 | |||
124 | // --------------------------------------------------------------------------- | ||
125 | |||
126 | function buildDigest (body: any) { | ||
127 | const rawBody = typeof body === 'string' ? body : JSON.stringify(body) | ||
128 | |||
129 | return 'SHA-256=' + sha256(rawBody, 'base64') | ||
130 | } | ||
131 | |||
132 | // --------------------------------------------------------------------------- | ||
133 | // Encryption | ||
134 | // --------------------------------------------------------------------------- | ||
135 | |||
136 | async function encrypt (str: string, secret: string) { | ||
137 | const iv = await randomBytesPromise(ENCRYPTION.IV) | ||
138 | |||
139 | const key = await scryptPromise(secret, ENCRYPTION.SALT, 32) | ||
140 | const cipher = createCipheriv(ENCRYPTION.ALGORITHM, key, iv) | ||
141 | |||
142 | let encrypted = iv.toString(ENCRYPTION.ENCODING) + ':' | ||
143 | encrypted += cipher.update(str, 'utf8', ENCRYPTION.ENCODING) | ||
144 | encrypted += cipher.final(ENCRYPTION.ENCODING) | ||
145 | |||
146 | return encrypted | ||
147 | } | ||
148 | |||
149 | async function decrypt (encryptedArg: string, secret: string) { | ||
150 | const [ ivStr, encryptedStr ] = encryptedArg.split(':') | ||
151 | |||
152 | const iv = Buffer.from(ivStr, 'hex') | ||
153 | const key = await scryptPromise(secret, ENCRYPTION.SALT, 32) | ||
154 | |||
155 | const decipher = createDecipheriv(ENCRYPTION.ALGORITHM, key, iv) | ||
156 | |||
157 | return decipher.update(encryptedStr, ENCRYPTION.ENCODING, 'utf8') + decipher.final('utf8') | ||
158 | } | ||
159 | |||
160 | // --------------------------------------------------------------------------- | ||
161 | |||
162 | export { | ||
163 | isHTTPSignatureDigestValid, | ||
164 | parseHTTPSignature, | ||
165 | isHTTPSignatureVerified, | ||
166 | buildDigest, | ||
167 | isJsonLDSignatureVerified, | ||
168 | comparePassword, | ||
169 | createPrivateAndPublicKeys, | ||
170 | cryptPassword, | ||
171 | signJsonLDObject, | ||
172 | |||
173 | encrypt, | ||
174 | decrypt | ||
175 | } | ||
176 | |||
177 | // --------------------------------------------------------------------------- | ||
178 | |||
179 | function hashObject (obj: any): Promise<any> { | ||
180 | return jsonld.promises.normalize(obj, { | ||
181 | safe: false, | ||
182 | algorithm: 'URDNA2015', | ||
183 | format: 'application/n-quads' | ||
184 | }).then(res => sha256(res)) | ||
185 | } | ||
186 | |||
187 | function createSignatureHash (signature: any) { | ||
188 | const signatureCopy = cloneDeep(signature) | ||
189 | Object.assign(signatureCopy, { | ||
190 | '@context': [ | ||
191 | 'https://w3id.org/security/v1', | ||
192 | { RsaSignature2017: 'https://w3id.org/security#RsaSignature2017' } | ||
193 | ] | ||
194 | }) | ||
195 | |||
196 | delete signatureCopy.type | ||
197 | delete signatureCopy.id | ||
198 | delete signatureCopy.signatureValue | ||
199 | |||
200 | return hashObject(signatureCopy) | ||
201 | } | ||
202 | |||
203 | function createDocWithoutSignatureHash (doc: any) { | ||
204 | const docWithoutSignature = cloneDeep(doc) | ||
205 | delete docWithoutSignature.signature | ||
206 | |||
207 | return hashObject(docWithoutSignature) | ||
208 | } | ||
diff --git a/server/helpers/promise-cache.ts b/server/helpers/promise-cache.ts deleted file mode 100644 index 303bab976..000000000 --- a/server/helpers/promise-cache.ts +++ /dev/null | |||
@@ -1,39 +0,0 @@ | |||
1 | export class CachePromiseFactory <A, R> { | ||
2 | private readonly running = new Map<string, Promise<R>>() | ||
3 | |||
4 | constructor ( | ||
5 | private readonly fn: (arg: A) => Promise<R>, | ||
6 | private readonly keyBuilder: (arg: A) => string | ||
7 | ) { | ||
8 | } | ||
9 | |||
10 | run (arg: A) { | ||
11 | return this.runWithContext(null, arg) | ||
12 | } | ||
13 | |||
14 | runWithContext (ctx: any, arg: A) { | ||
15 | const key = this.keyBuilder(arg) | ||
16 | |||
17 | if (this.running.has(key)) return this.running.get(key) | ||
18 | |||
19 | const p = this.fn.apply(ctx || this, [ arg ]) | ||
20 | |||
21 | this.running.set(key, p) | ||
22 | |||
23 | return p.finally(() => this.running.delete(key)) | ||
24 | } | ||
25 | } | ||
26 | |||
27 | export function CachePromise (options: { | ||
28 | keyBuilder: (...args: any[]) => string | ||
29 | }) { | ||
30 | return function (_target, _key, descriptor: PropertyDescriptor) { | ||
31 | const promiseCache = new CachePromiseFactory(descriptor.value, options.keyBuilder) | ||
32 | |||
33 | descriptor.value = function () { | ||
34 | if (arguments.length !== 1) throw new Error('Cache promise only support methods with 1 argument') | ||
35 | |||
36 | return promiseCache.runWithContext(this, arguments[0]) | ||
37 | } | ||
38 | } | ||
39 | } | ||
diff --git a/server/helpers/proxy.ts b/server/helpers/proxy.ts deleted file mode 100644 index 8b82ccae0..000000000 --- a/server/helpers/proxy.ts +++ /dev/null | |||
@@ -1,14 +0,0 @@ | |||
1 | function getProxy () { | ||
2 | return process.env.HTTPS_PROXY || | ||
3 | process.env.HTTP_PROXY || | ||
4 | undefined | ||
5 | } | ||
6 | |||
7 | function isProxyEnabled () { | ||
8 | return !!getProxy() | ||
9 | } | ||
10 | |||
11 | export { | ||
12 | getProxy, | ||
13 | isProxyEnabled | ||
14 | } | ||
diff --git a/server/helpers/query.ts b/server/helpers/query.ts deleted file mode 100644 index c0f78368f..000000000 --- a/server/helpers/query.ts +++ /dev/null | |||
@@ -1,81 +0,0 @@ | |||
1 | import { pick } from '@shared/core-utils' | ||
2 | import { | ||
3 | VideoChannelsSearchQueryAfterSanitize, | ||
4 | VideoPlaylistsSearchQueryAfterSanitize, | ||
5 | VideosCommonQueryAfterSanitize, | ||
6 | VideosSearchQueryAfterSanitize | ||
7 | } from '@shared/models' | ||
8 | |||
9 | function pickCommonVideoQuery (query: VideosCommonQueryAfterSanitize) { | ||
10 | return pick(query, [ | ||
11 | 'start', | ||
12 | 'count', | ||
13 | 'sort', | ||
14 | 'nsfw', | ||
15 | 'isLive', | ||
16 | 'categoryOneOf', | ||
17 | 'licenceOneOf', | ||
18 | 'languageOneOf', | ||
19 | 'privacyOneOf', | ||
20 | 'tagsOneOf', | ||
21 | 'tagsAllOf', | ||
22 | 'isLocal', | ||
23 | 'include', | ||
24 | 'skipCount', | ||
25 | 'hasHLSFiles', | ||
26 | 'hasWebtorrentFiles', // TODO: Remove in v7 | ||
27 | 'hasWebVideoFiles', | ||
28 | 'search', | ||
29 | 'excludeAlreadyWatched' | ||
30 | ]) | ||
31 | } | ||
32 | |||
33 | function pickSearchVideoQuery (query: VideosSearchQueryAfterSanitize) { | ||
34 | return { | ||
35 | ...pickCommonVideoQuery(query), | ||
36 | |||
37 | ...pick(query, [ | ||
38 | 'searchTarget', | ||
39 | 'host', | ||
40 | 'startDate', | ||
41 | 'endDate', | ||
42 | 'originallyPublishedStartDate', | ||
43 | 'originallyPublishedEndDate', | ||
44 | 'durationMin', | ||
45 | 'durationMax', | ||
46 | 'uuids', | ||
47 | 'excludeAlreadyWatched' | ||
48 | ]) | ||
49 | } | ||
50 | } | ||
51 | |||
52 | function pickSearchChannelQuery (query: VideoChannelsSearchQueryAfterSanitize) { | ||
53 | return pick(query, [ | ||
54 | 'searchTarget', | ||
55 | 'search', | ||
56 | 'start', | ||
57 | 'count', | ||
58 | 'sort', | ||
59 | 'host', | ||
60 | 'handles' | ||
61 | ]) | ||
62 | } | ||
63 | |||
64 | function pickSearchPlaylistQuery (query: VideoPlaylistsSearchQueryAfterSanitize) { | ||
65 | return pick(query, [ | ||
66 | 'searchTarget', | ||
67 | 'search', | ||
68 | 'start', | ||
69 | 'count', | ||
70 | 'sort', | ||
71 | 'host', | ||
72 | 'uuids' | ||
73 | ]) | ||
74 | } | ||
75 | |||
76 | export { | ||
77 | pickCommonVideoQuery, | ||
78 | pickSearchVideoQuery, | ||
79 | pickSearchPlaylistQuery, | ||
80 | pickSearchChannelQuery | ||
81 | } | ||
diff --git a/server/helpers/regexp.ts b/server/helpers/regexp.ts deleted file mode 100644 index 257054cea..000000000 --- a/server/helpers/regexp.ts +++ /dev/null | |||
@@ -1,22 +0,0 @@ | |||
1 | // Thanks to https://regex101.com | ||
2 | function regexpCapture (str: string, regex: RegExp, maxIterations = 100) { | ||
3 | const result: RegExpExecArray[] = [] | ||
4 | let m: RegExpExecArray | ||
5 | let i = 0 | ||
6 | |||
7 | while ((m = regex.exec(str)) !== null && i < maxIterations) { | ||
8 | // This is necessary to avoid infinite loops with zero-width matches | ||
9 | if (m.index === regex.lastIndex) { | ||
10 | regex.lastIndex++ | ||
11 | } | ||
12 | |||
13 | result.push(m) | ||
14 | i++ | ||
15 | } | ||
16 | |||
17 | return result | ||
18 | } | ||
19 | |||
20 | export { | ||
21 | regexpCapture | ||
22 | } | ||
diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts deleted file mode 100644 index 1625d6e49..000000000 --- a/server/helpers/requests.ts +++ /dev/null | |||
@@ -1,246 +0,0 @@ | |||
1 | import { createWriteStream, remove } from 'fs-extra' | ||
2 | import got, { CancelableRequest, NormalizedOptions, Options as GotOptions, RequestError, Response } from 'got' | ||
3 | import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent' | ||
4 | import { ACTIVITY_PUB, BINARY_CONTENT_TYPES, PEERTUBE_VERSION, REQUEST_TIMEOUTS, WEBSERVER } from '../initializers/constants' | ||
5 | import { pipelinePromise } from './core-utils' | ||
6 | import { logger, loggerTagsFactory } from './logger' | ||
7 | import { getProxy, isProxyEnabled } from './proxy' | ||
8 | |||
9 | const lTags = loggerTagsFactory('request') | ||
10 | |||
11 | const httpSignature = require('@peertube/http-signature') | ||
12 | |||
13 | export interface PeerTubeRequestError extends Error { | ||
14 | statusCode?: number | ||
15 | responseBody?: any | ||
16 | responseHeaders?: any | ||
17 | requestHeaders?: any | ||
18 | } | ||
19 | |||
20 | type PeerTubeRequestOptions = { | ||
21 | timeout?: number | ||
22 | activityPub?: boolean | ||
23 | bodyKBLimit?: number // 1MB | ||
24 | |||
25 | httpSignature?: { | ||
26 | algorithm: string | ||
27 | authorizationHeaderName: string | ||
28 | keyId: string | ||
29 | key: string | ||
30 | headers: string[] | ||
31 | } | ||
32 | |||
33 | jsonResponse?: boolean | ||
34 | |||
35 | followRedirect?: boolean | ||
36 | } & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'> | ||
37 | |||
38 | const peertubeGot = got.extend({ | ||
39 | ...getAgent(), | ||
40 | |||
41 | headers: { | ||
42 | 'user-agent': getUserAgent() | ||
43 | }, | ||
44 | |||
45 | handlers: [ | ||
46 | (options, next) => { | ||
47 | const promiseOrStream = next(options) as CancelableRequest<any> | ||
48 | const bodyKBLimit = options.context?.bodyKBLimit as number | ||
49 | if (!bodyKBLimit) throw new Error('No KB limit for this request') | ||
50 | |||
51 | const bodyLimit = bodyKBLimit * 1000 | ||
52 | |||
53 | /* eslint-disable @typescript-eslint/no-floating-promises */ | ||
54 | promiseOrStream.on('downloadProgress', progress => { | ||
55 | if (progress.transferred > bodyLimit && progress.percent !== 1) { | ||
56 | const message = `Exceeded the download limit of ${bodyLimit} B` | ||
57 | logger.warn(message, lTags()) | ||
58 | |||
59 | // CancelableRequest | ||
60 | if (promiseOrStream.cancel) { | ||
61 | promiseOrStream.cancel() | ||
62 | return | ||
63 | } | ||
64 | |||
65 | // Stream | ||
66 | (promiseOrStream as any).destroy() | ||
67 | } | ||
68 | }) | ||
69 | |||
70 | return promiseOrStream | ||
71 | } | ||
72 | ], | ||
73 | |||
74 | hooks: { | ||
75 | beforeRequest: [ | ||
76 | options => { | ||
77 | const headers = options.headers || {} | ||
78 | headers['host'] = options.url.host | ||
79 | }, | ||
80 | |||
81 | options => { | ||
82 | const httpSignatureOptions = options.context?.httpSignature | ||
83 | |||
84 | if (httpSignatureOptions) { | ||
85 | const method = options.method ?? 'GET' | ||
86 | const path = options.path ?? options.url.pathname | ||
87 | |||
88 | if (!method || !path) { | ||
89 | throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`) | ||
90 | } | ||
91 | |||
92 | httpSignature.signRequest({ | ||
93 | getHeader: function (header: string) { | ||
94 | const value = options.headers[header.toLowerCase()] | ||
95 | |||
96 | if (!value) logger.warn('Unknown header requested by http-signature.', { headers: options.headers, header }) | ||
97 | return value | ||
98 | }, | ||
99 | |||
100 | setHeader: function (header: string, value: string) { | ||
101 | options.headers[header] = value | ||
102 | }, | ||
103 | |||
104 | method, | ||
105 | path | ||
106 | }, httpSignatureOptions) | ||
107 | } | ||
108 | } | ||
109 | ], | ||
110 | |||
111 | beforeRetry: [ | ||
112 | (_options: NormalizedOptions, error: RequestError, retryCount: number) => { | ||
113 | logger.debug('Retrying request to %s.', error.request.requestUrl, { retryCount, error: buildRequestError(error), ...lTags() }) | ||
114 | } | ||
115 | ] | ||
116 | } | ||
117 | }) | ||
118 | |||
119 | function doRequest (url: string, options: PeerTubeRequestOptions = {}) { | ||
120 | const gotOptions = buildGotOptions(options) | ||
121 | |||
122 | return peertubeGot(url, gotOptions) | ||
123 | .catch(err => { throw buildRequestError(err) }) | ||
124 | } | ||
125 | |||
126 | function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) { | ||
127 | const gotOptions = buildGotOptions(options) | ||
128 | |||
129 | return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' }) | ||
130 | .catch(err => { throw buildRequestError(err) }) | ||
131 | } | ||
132 | |||
133 | async function doRequestAndSaveToFile ( | ||
134 | url: string, | ||
135 | destPath: string, | ||
136 | options: PeerTubeRequestOptions = {} | ||
137 | ) { | ||
138 | const gotOptions = buildGotOptions({ ...options, timeout: options.timeout ?? REQUEST_TIMEOUTS.FILE }) | ||
139 | |||
140 | const outFile = createWriteStream(destPath) | ||
141 | |||
142 | try { | ||
143 | await pipelinePromise( | ||
144 | peertubeGot.stream(url, gotOptions), | ||
145 | outFile | ||
146 | ) | ||
147 | } catch (err) { | ||
148 | remove(destPath) | ||
149 | .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err, ...lTags() })) | ||
150 | |||
151 | throw buildRequestError(err) | ||
152 | } | ||
153 | } | ||
154 | |||
155 | function getAgent () { | ||
156 | if (!isProxyEnabled()) return {} | ||
157 | |||
158 | const proxy = getProxy() | ||
159 | |||
160 | logger.info('Using proxy %s.', proxy, lTags()) | ||
161 | |||
162 | const proxyAgentOptions = { | ||
163 | keepAlive: true, | ||
164 | keepAliveMsecs: 1000, | ||
165 | maxSockets: 256, | ||
166 | maxFreeSockets: 256, | ||
167 | scheduling: 'lifo' as 'lifo', | ||
168 | proxy | ||
169 | } | ||
170 | |||
171 | return { | ||
172 | agent: { | ||
173 | http: new HttpProxyAgent(proxyAgentOptions), | ||
174 | https: new HttpsProxyAgent(proxyAgentOptions) | ||
175 | } | ||
176 | } | ||
177 | } | ||
178 | |||
179 | function getUserAgent () { | ||
180 | return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})` | ||
181 | } | ||
182 | |||
183 | function isBinaryResponse (result: Response<any>) { | ||
184 | return BINARY_CONTENT_TYPES.has(result.headers['content-type']) | ||
185 | } | ||
186 | |||
187 | // --------------------------------------------------------------------------- | ||
188 | |||
189 | export { | ||
190 | PeerTubeRequestOptions, | ||
191 | |||
192 | doRequest, | ||
193 | doJSONRequest, | ||
194 | doRequestAndSaveToFile, | ||
195 | isBinaryResponse, | ||
196 | getAgent, | ||
197 | peertubeGot | ||
198 | } | ||
199 | |||
200 | // --------------------------------------------------------------------------- | ||
201 | |||
202 | function buildGotOptions (options: PeerTubeRequestOptions) { | ||
203 | const { activityPub, bodyKBLimit = 1000 } = options | ||
204 | |||
205 | const context = { bodyKBLimit, httpSignature: options.httpSignature } | ||
206 | |||
207 | let headers = options.headers || {} | ||
208 | |||
209 | if (!headers.date) { | ||
210 | headers = { ...headers, date: new Date().toUTCString() } | ||
211 | } | ||
212 | |||
213 | if (activityPub && !headers.accept) { | ||
214 | headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER } | ||
215 | } | ||
216 | |||
217 | return { | ||
218 | method: options.method, | ||
219 | dnsCache: true, | ||
220 | timeout: options.timeout ?? REQUEST_TIMEOUTS.DEFAULT, | ||
221 | json: options.json, | ||
222 | searchParams: options.searchParams, | ||
223 | followRedirect: options.followRedirect, | ||
224 | retry: 2, | ||
225 | headers, | ||
226 | context | ||
227 | } | ||
228 | } | ||
229 | |||
230 | function buildRequestError (error: RequestError) { | ||
231 | const newError: PeerTubeRequestError = new Error(error.message) | ||
232 | newError.name = error.name | ||
233 | newError.stack = error.stack | ||
234 | |||
235 | if (error.response) { | ||
236 | newError.responseBody = error.response.body | ||
237 | newError.responseHeaders = error.response.headers | ||
238 | newError.statusCode = error.response.statusCode | ||
239 | } | ||
240 | |||
241 | if (error.options) { | ||
242 | newError.requestHeaders = error.options.headers | ||
243 | } | ||
244 | |||
245 | return newError | ||
246 | } | ||
diff --git a/server/helpers/stream-replacer.ts b/server/helpers/stream-replacer.ts deleted file mode 100644 index 4babab418..000000000 --- a/server/helpers/stream-replacer.ts +++ /dev/null | |||
@@ -1,58 +0,0 @@ | |||
1 | import { Transform, TransformCallback } from 'stream' | ||
2 | |||
3 | // Thanks: https://stackoverflow.com/a/45126242 | ||
4 | class StreamReplacer extends Transform { | ||
5 | private pendingChunk: Buffer | ||
6 | |||
7 | constructor (private readonly replacer: (line: string) => string) { | ||
8 | super() | ||
9 | } | ||
10 | |||
11 | _transform (chunk: Buffer, _encoding: BufferEncoding, done: TransformCallback) { | ||
12 | try { | ||
13 | this.pendingChunk = this.pendingChunk?.length | ||
14 | ? Buffer.concat([ this.pendingChunk, chunk ]) | ||
15 | : chunk | ||
16 | |||
17 | let index: number | ||
18 | |||
19 | // As long as we keep finding newlines, keep making slices of the buffer and push them to the | ||
20 | // readable side of the transform stream | ||
21 | while ((index = this.pendingChunk.indexOf('\n')) !== -1) { | ||
22 | // The `end` parameter is non-inclusive, so increase it to include the newline we found | ||
23 | const line = this.pendingChunk.slice(0, ++index) | ||
24 | |||
25 | // `start` is inclusive, but we are already one char ahead of the newline -> all good | ||
26 | this.pendingChunk = this.pendingChunk.slice(index) | ||
27 | |||
28 | // We have a single line here! Prepend the string we want | ||
29 | this.push(this.doReplace(line)) | ||
30 | } | ||
31 | |||
32 | return done() | ||
33 | } catch (err) { | ||
34 | return done(err) | ||
35 | } | ||
36 | } | ||
37 | |||
38 | _flush (done: TransformCallback) { | ||
39 | // If we have any remaining data in the cache, send it out | ||
40 | if (!this.pendingChunk?.length) return done() | ||
41 | |||
42 | try { | ||
43 | return done(null, this.doReplace(this.pendingChunk)) | ||
44 | } catch (err) { | ||
45 | return done(err) | ||
46 | } | ||
47 | } | ||
48 | |||
49 | private doReplace (buffer: Buffer) { | ||
50 | const line = this.replacer(buffer.toString('utf8')) | ||
51 | |||
52 | return Buffer.from(line, 'utf8') | ||
53 | } | ||
54 | } | ||
55 | |||
56 | export { | ||
57 | StreamReplacer | ||
58 | } | ||
diff --git a/server/helpers/token-generator.ts b/server/helpers/token-generator.ts deleted file mode 100644 index 16313b818..000000000 --- a/server/helpers/token-generator.ts +++ /dev/null | |||
@@ -1,19 +0,0 @@ | |||
1 | import { buildUUID } from '@shared/extra-utils' | ||
2 | |||
3 | function generateRunnerRegistrationToken () { | ||
4 | return 'ptrrt-' + buildUUID() | ||
5 | } | ||
6 | |||
7 | function generateRunnerToken () { | ||
8 | return 'ptrt-' + buildUUID() | ||
9 | } | ||
10 | |||
11 | function generateRunnerJobToken () { | ||
12 | return 'ptrjt-' + buildUUID() | ||
13 | } | ||
14 | |||
15 | export { | ||
16 | generateRunnerRegistrationToken, | ||
17 | generateRunnerToken, | ||
18 | generateRunnerJobToken | ||
19 | } | ||
diff --git a/server/helpers/upload.ts b/server/helpers/upload.ts deleted file mode 100644 index f5f476913..000000000 --- a/server/helpers/upload.ts +++ /dev/null | |||
@@ -1,14 +0,0 @@ | |||
1 | import { join } from 'path' | ||
2 | import { DIRECTORIES } from '@server/initializers/constants' | ||
3 | |||
4 | function getResumableUploadPath (filename?: string) { | ||
5 | if (filename) return join(DIRECTORIES.RESUMABLE_UPLOAD, filename) | ||
6 | |||
7 | return DIRECTORIES.RESUMABLE_UPLOAD | ||
8 | } | ||
9 | |||
10 | // --------------------------------------------------------------------------- | ||
11 | |||
12 | export { | ||
13 | getResumableUploadPath | ||
14 | } | ||
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts deleted file mode 100644 index 5a4fe4fdd..000000000 --- a/server/helpers/utils.ts +++ /dev/null | |||
@@ -1,70 +0,0 @@ | |||
1 | import { remove } from 'fs-extra' | ||
2 | import { Instance as ParseTorrent } from 'parse-torrent' | ||
3 | import { join } from 'path' | ||
4 | import { sha256 } from '@shared/extra-utils' | ||
5 | import { ResultList } from '@shared/models' | ||
6 | import { CONFIG } from '../initializers/config' | ||
7 | import { randomBytesPromise } from './core-utils' | ||
8 | import { logger } from './logger' | ||
9 | |||
10 | function deleteFileAndCatch (path: string) { | ||
11 | remove(path) | ||
12 | .catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err })) | ||
13 | } | ||
14 | |||
15 | async function generateRandomString (size: number) { | ||
16 | const raw = await randomBytesPromise(size) | ||
17 | |||
18 | return raw.toString('hex') | ||
19 | } | ||
20 | |||
21 | interface FormattableToJSON<U, V> { | ||
22 | toFormattedJSON (args?: U): V | ||
23 | } | ||
24 | |||
25 | function getFormattedObjects<U, V, T extends FormattableToJSON<U, V>> (objects: T[], objectsTotal: number, formattedArg?: U) { | ||
26 | const formattedObjects = objects.map(o => o.toFormattedJSON(formattedArg)) | ||
27 | |||
28 | return { | ||
29 | total: objectsTotal, | ||
30 | data: formattedObjects | ||
31 | } as ResultList<V> | ||
32 | } | ||
33 | |||
34 | function generateVideoImportTmpPath (target: string | ParseTorrent, extension = '.mp4') { | ||
35 | const id = typeof target === 'string' | ||
36 | ? target | ||
37 | : target.infoHash | ||
38 | |||
39 | const hash = sha256(id) | ||
40 | return join(CONFIG.STORAGE.TMP_DIR, `${hash}-import${extension}`) | ||
41 | } | ||
42 | |||
43 | function getSecureTorrentName (originalName: string) { | ||
44 | return sha256(originalName) + '.torrent' | ||
45 | } | ||
46 | |||
47 | /** | ||
48 | * From a filename like "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3.mp4", returns | ||
49 | * only the "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3" part. If the filename does | ||
50 | * not contain a UUID, returns null. | ||
51 | */ | ||
52 | function getUUIDFromFilename (filename: string) { | ||
53 | const regex = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/ | ||
54 | const result = filename.match(regex) | ||
55 | |||
56 | if (!result || Array.isArray(result) === false) return null | ||
57 | |||
58 | return result[0] | ||
59 | } | ||
60 | |||
61 | // --------------------------------------------------------------------------- | ||
62 | |||
63 | export { | ||
64 | deleteFileAndCatch, | ||
65 | generateRandomString, | ||
66 | getFormattedObjects, | ||
67 | getSecureTorrentName, | ||
68 | generateVideoImportTmpPath, | ||
69 | getUUIDFromFilename | ||
70 | } | ||
diff --git a/server/helpers/version.ts b/server/helpers/version.ts deleted file mode 100644 index 5b3bf59dd..000000000 --- a/server/helpers/version.ts +++ /dev/null | |||
@@ -1,36 +0,0 @@ | |||
1 | import { execPromise, execPromise2 } from './core-utils' | ||
2 | import { logger } from './logger' | ||
3 | |||
4 | async function getServerCommit () { | ||
5 | try { | ||
6 | const tag = await execPromise2( | ||
7 | '[ ! -d .git ] || git name-rev --name-only --tags --no-undefined HEAD 2>/dev/null || true', | ||
8 | { stdio: [ 0, 1, 2 ] } | ||
9 | ) | ||
10 | |||
11 | if (tag) return tag.replace(/^v/, '') | ||
12 | } catch (err) { | ||
13 | logger.debug('Cannot get version from git tags.', { err }) | ||
14 | } | ||
15 | |||
16 | try { | ||
17 | const version = await execPromise('[ ! -d .git ] || git rev-parse --short HEAD') | ||
18 | |||
19 | if (version) return version.toString().trim() | ||
20 | } catch (err) { | ||
21 | logger.debug('Cannot get version from git HEAD.', { err }) | ||
22 | } | ||
23 | |||
24 | return '' | ||
25 | } | ||
26 | |||
27 | function getNodeABIVersion () { | ||
28 | const version = process.versions.modules | ||
29 | |||
30 | return parseInt(version) | ||
31 | } | ||
32 | |||
33 | export { | ||
34 | getServerCommit, | ||
35 | getNodeABIVersion | ||
36 | } | ||
diff --git a/server/helpers/video.ts b/server/helpers/video.ts deleted file mode 100644 index c688ef1e3..000000000 --- a/server/helpers/video.ts +++ /dev/null | |||
@@ -1,51 +0,0 @@ | |||
1 | import { Response } from 'express' | ||
2 | import { CONFIG } from '@server/initializers/config' | ||
3 | import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models' | ||
4 | import { VideoPrivacy, VideoState } from '@shared/models' | ||
5 | import { forceNumber } from '@shared/core-utils' | ||
6 | |||
7 | function getVideoWithAttributes (res: Response) { | ||
8 | return res.locals.videoAPI || res.locals.videoAll || res.locals.onlyVideo | ||
9 | } | ||
10 | |||
11 | function extractVideo (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) { | ||
12 | return isStreamingPlaylist(videoOrPlaylist) | ||
13 | ? videoOrPlaylist.Video | ||
14 | : videoOrPlaylist | ||
15 | } | ||
16 | |||
17 | function isPrivacyForFederation (privacy: VideoPrivacy) { | ||
18 | const castedPrivacy = forceNumber(privacy) | ||
19 | |||
20 | return castedPrivacy === VideoPrivacy.PUBLIC || | ||
21 | (CONFIG.FEDERATION.VIDEOS.FEDERATE_UNLISTED === true && castedPrivacy === VideoPrivacy.UNLISTED) | ||
22 | } | ||
23 | |||
24 | function isStateForFederation (state: VideoState) { | ||
25 | const castedState = forceNumber(state) | ||
26 | |||
27 | return castedState === VideoState.PUBLISHED || castedState === VideoState.WAITING_FOR_LIVE || castedState === VideoState.LIVE_ENDED | ||
28 | } | ||
29 | |||
30 | function getPrivaciesForFederation () { | ||
31 | return (CONFIG.FEDERATION.VIDEOS.FEDERATE_UNLISTED === true) | ||
32 | ? [ { privacy: VideoPrivacy.PUBLIC }, { privacy: VideoPrivacy.UNLISTED } ] | ||
33 | : [ { privacy: VideoPrivacy.PUBLIC } ] | ||
34 | } | ||
35 | |||
36 | function getExtFromMimetype (mimeTypes: { [id: string]: string | string[] }, mimeType: string) { | ||
37 | const value = mimeTypes[mimeType] | ||
38 | |||
39 | if (Array.isArray(value)) return value[0] | ||
40 | |||
41 | return value | ||
42 | } | ||
43 | |||
44 | export { | ||
45 | getVideoWithAttributes, | ||
46 | extractVideo, | ||
47 | getExtFromMimetype, | ||
48 | isStateForFederation, | ||
49 | isPrivacyForFederation, | ||
50 | getPrivaciesForFederation | ||
51 | } | ||
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts deleted file mode 100644 index f33a7bccd..000000000 --- a/server/helpers/webtorrent.ts +++ /dev/null | |||
@@ -1,258 +0,0 @@ | |||
1 | import { decode, encode } from 'bencode' | ||
2 | import createTorrent from 'create-torrent' | ||
3 | import { createWriteStream, ensureDir, pathExists, readFile, remove, writeFile } from 'fs-extra' | ||
4 | import { encode as magnetUriEncode } from 'magnet-uri' | ||
5 | import parseTorrent from 'parse-torrent' | ||
6 | import { dirname, join } from 'path' | ||
7 | import { pipeline } from 'stream' | ||
8 | import WebTorrent, { Instance, TorrentFile } from 'webtorrent' | ||
9 | import { isArray } from '@server/helpers/custom-validators/misc' | ||
10 | import { WEBSERVER } from '@server/initializers/constants' | ||
11 | import { generateTorrentFileName } from '@server/lib/paths' | ||
12 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
13 | import { MVideo } from '@server/types/models/video/video' | ||
14 | import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file' | ||
15 | import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist' | ||
16 | import { promisify2 } from '@shared/core-utils' | ||
17 | import { sha1 } from '@shared/extra-utils' | ||
18 | import { CONFIG } from '../initializers/config' | ||
19 | import { logger } from './logger' | ||
20 | import { generateVideoImportTmpPath } from './utils' | ||
21 | import { extractVideo } from './video' | ||
22 | |||
23 | const createTorrentPromise = promisify2<string, any, any>(createTorrent) | ||
24 | |||
25 | async function downloadWebTorrentVideo (target: { uri: string, torrentName?: string }, timeout: number) { | ||
26 | const id = target.uri || target.torrentName | ||
27 | let timer | ||
28 | |||
29 | const path = generateVideoImportTmpPath(id) | ||
30 | logger.info('Importing torrent video %s', id) | ||
31 | |||
32 | const directoryPath = join(CONFIG.STORAGE.TMP_DIR, 'webtorrent') | ||
33 | await ensureDir(directoryPath) | ||
34 | |||
35 | return new Promise<string>((res, rej) => { | ||
36 | const webtorrent = new WebTorrent() | ||
37 | let file: TorrentFile | ||
38 | |||
39 | const torrentId = target.uri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName) | ||
40 | |||
41 | const options = { path: directoryPath } | ||
42 | const torrent = webtorrent.add(torrentId, options, torrent => { | ||
43 | if (torrent.files.length !== 1) { | ||
44 | if (timer) clearTimeout(timer) | ||
45 | |||
46 | for (const file of torrent.files) { | ||
47 | deleteDownloadedFile({ directoryPath, filepath: file.path }) | ||
48 | } | ||
49 | |||
50 | return safeWebtorrentDestroy(webtorrent, torrentId, undefined, target.torrentName) | ||
51 | .then(() => rej(new Error('Cannot import torrent ' + torrentId + ': there are multiple files in it'))) | ||
52 | } | ||
53 | |||
54 | logger.debug('Got torrent from webtorrent %s.', id, { infoHash: torrent.infoHash }) | ||
55 | |||
56 | file = torrent.files[0] | ||
57 | |||
58 | // FIXME: avoid creating another stream when https://github.com/webtorrent/webtorrent/issues/1517 is fixed | ||
59 | const writeStream = createWriteStream(path) | ||
60 | writeStream.on('finish', () => { | ||
61 | if (timer) clearTimeout(timer) | ||
62 | |||
63 | safeWebtorrentDestroy(webtorrent, torrentId, { directoryPath, filepath: file.path }, target.torrentName) | ||
64 | .then(() => res(path)) | ||
65 | .catch(err => logger.error('Cannot destroy webtorrent.', { err })) | ||
66 | }) | ||
67 | |||
68 | pipeline( | ||
69 | file.createReadStream(), | ||
70 | writeStream, | ||
71 | err => { | ||
72 | if (err) rej(err) | ||
73 | } | ||
74 | ) | ||
75 | }) | ||
76 | |||
77 | torrent.on('error', err => rej(err)) | ||
78 | |||
79 | timer = setTimeout(() => { | ||
80 | const err = new Error('Webtorrent download timeout.') | ||
81 | |||
82 | safeWebtorrentDestroy(webtorrent, torrentId, file ? { directoryPath, filepath: file.path } : undefined, target.torrentName) | ||
83 | .then(() => rej(err)) | ||
84 | .catch(destroyErr => { | ||
85 | logger.error('Cannot destroy webtorrent.', { err: destroyErr }) | ||
86 | rej(err) | ||
87 | }) | ||
88 | |||
89 | }, timeout) | ||
90 | }) | ||
91 | } | ||
92 | |||
93 | function createTorrentAndSetInfoHash (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { | ||
94 | return VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(videoOrPlaylist), videoPath => { | ||
95 | return createTorrentAndSetInfoHashFromPath(videoOrPlaylist, videoFile, videoPath) | ||
96 | }) | ||
97 | } | ||
98 | |||
99 | async function createTorrentAndSetInfoHashFromPath ( | ||
100 | videoOrPlaylist: MVideo | MStreamingPlaylistVideo, | ||
101 | videoFile: MVideoFile, | ||
102 | filePath: string | ||
103 | ) { | ||
104 | const video = extractVideo(videoOrPlaylist) | ||
105 | |||
106 | const options = { | ||
107 | // Keep the extname, it's used by the client to stream the file inside a web browser | ||
108 | name: buildInfoName(video, videoFile), | ||
109 | createdBy: 'PeerTube', | ||
110 | announceList: buildAnnounceList(), | ||
111 | urlList: buildUrlList(video, videoFile) | ||
112 | } | ||
113 | |||
114 | const torrentContent = await createTorrentPromise(filePath, options) | ||
115 | |||
116 | const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) | ||
117 | const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename) | ||
118 | logger.info('Creating torrent %s.', torrentPath) | ||
119 | |||
120 | await writeFile(torrentPath, torrentContent) | ||
121 | |||
122 | // Remove old torrent file if it existed | ||
123 | if (videoFile.hasTorrent()) { | ||
124 | await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)) | ||
125 | } | ||
126 | |||
127 | const parsedTorrent = parseTorrent(torrentContent) | ||
128 | videoFile.infoHash = parsedTorrent.infoHash | ||
129 | videoFile.torrentFilename = torrentFilename | ||
130 | } | ||
131 | |||
132 | async function updateTorrentMetadata (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { | ||
133 | const video = extractVideo(videoOrPlaylist) | ||
134 | |||
135 | const oldTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) | ||
136 | |||
137 | if (!await pathExists(oldTorrentPath)) { | ||
138 | logger.info('Do not update torrent metadata %s of video %s because the file does not exist anymore.', video.uuid, oldTorrentPath) | ||
139 | return | ||
140 | } | ||
141 | |||
142 | const torrentContent = await readFile(oldTorrentPath) | ||
143 | const decoded = decode(torrentContent) | ||
144 | |||
145 | decoded['announce-list'] = buildAnnounceList() | ||
146 | decoded.announce = decoded['announce-list'][0][0] | ||
147 | |||
148 | decoded['url-list'] = buildUrlList(video, videoFile) | ||
149 | |||
150 | decoded.info.name = buildInfoName(video, videoFile) | ||
151 | decoded['creation date'] = Math.ceil(Date.now() / 1000) | ||
152 | |||
153 | const newTorrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) | ||
154 | const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, newTorrentFilename) | ||
155 | |||
156 | logger.info('Updating torrent metadata %s -> %s.', oldTorrentPath, newTorrentPath) | ||
157 | |||
158 | await writeFile(newTorrentPath, encode(decoded)) | ||
159 | await remove(oldTorrentPath) | ||
160 | |||
161 | videoFile.torrentFilename = newTorrentFilename | ||
162 | videoFile.infoHash = sha1(encode(decoded.info)) | ||
163 | } | ||
164 | |||
165 | function generateMagnetUri ( | ||
166 | video: MVideo, | ||
167 | videoFile: MVideoFileRedundanciesOpt, | ||
168 | trackerUrls: string[] | ||
169 | ) { | ||
170 | const xs = videoFile.getTorrentUrl() | ||
171 | const announce = trackerUrls | ||
172 | |||
173 | let urlList = video.hasPrivateStaticPath() | ||
174 | ? [] | ||
175 | : [ videoFile.getFileUrl(video) ] | ||
176 | |||
177 | const redundancies = videoFile.RedundancyVideos | ||
178 | if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl)) | ||
179 | |||
180 | const magnetHash = { | ||
181 | xs, | ||
182 | announce, | ||
183 | urlList, | ||
184 | infoHash: videoFile.infoHash, | ||
185 | name: video.name | ||
186 | } | ||
187 | |||
188 | return magnetUriEncode(magnetHash) | ||
189 | } | ||
190 | |||
191 | // --------------------------------------------------------------------------- | ||
192 | |||
193 | export { | ||
194 | createTorrentPromise, | ||
195 | updateTorrentMetadata, | ||
196 | |||
197 | createTorrentAndSetInfoHash, | ||
198 | createTorrentAndSetInfoHashFromPath, | ||
199 | |||
200 | generateMagnetUri, | ||
201 | downloadWebTorrentVideo | ||
202 | } | ||
203 | |||
204 | // --------------------------------------------------------------------------- | ||
205 | |||
206 | function safeWebtorrentDestroy ( | ||
207 | webtorrent: Instance, | ||
208 | torrentId: string, | ||
209 | downloadedFile?: { directoryPath: string, filepath: string }, | ||
210 | torrentName?: string | ||
211 | ) { | ||
212 | return new Promise<void>(res => { | ||
213 | webtorrent.destroy(err => { | ||
214 | // Delete torrent file | ||
215 | if (torrentName) { | ||
216 | logger.debug('Removing %s torrent after webtorrent download.', torrentId) | ||
217 | remove(torrentId) | ||
218 | .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err })) | ||
219 | } | ||
220 | |||
221 | // Delete downloaded file | ||
222 | if (downloadedFile) deleteDownloadedFile(downloadedFile) | ||
223 | |||
224 | if (err) logger.warn('Cannot destroy webtorrent in timeout.', { err }) | ||
225 | |||
226 | return res() | ||
227 | }) | ||
228 | }) | ||
229 | } | ||
230 | |||
231 | function deleteDownloadedFile (downloadedFile: { directoryPath: string, filepath: string }) { | ||
232 | // We want to delete the base directory | ||
233 | let pathToDelete = dirname(downloadedFile.filepath) | ||
234 | if (pathToDelete === '.') pathToDelete = downloadedFile.filepath | ||
235 | |||
236 | const toRemovePath = join(downloadedFile.directoryPath, pathToDelete) | ||
237 | |||
238 | logger.debug('Removing %s after webtorrent download.', toRemovePath) | ||
239 | remove(toRemovePath) | ||
240 | .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', toRemovePath, { err })) | ||
241 | } | ||
242 | |||
243 | function buildAnnounceList () { | ||
244 | return [ | ||
245 | [ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ], | ||
246 | [ WEBSERVER.URL + '/tracker/announce' ] | ||
247 | ] | ||
248 | } | ||
249 | |||
250 | function buildUrlList (video: MVideo, videoFile: MVideoFile) { | ||
251 | if (video.hasPrivateStaticPath()) return [] | ||
252 | |||
253 | return [ videoFile.getFileUrl(video) ] | ||
254 | } | ||
255 | |||
256 | function buildInfoName (video: MVideo, videoFile: MVideoFile) { | ||
257 | return `${video.name} ${videoFile.resolution}p${videoFile.extname}` | ||
258 | } | ||
diff --git a/server/helpers/youtube-dl/index.ts b/server/helpers/youtube-dl/index.ts deleted file mode 100644 index 6afc77dcf..000000000 --- a/server/helpers/youtube-dl/index.ts +++ /dev/null | |||
@@ -1,3 +0,0 @@ | |||
1 | export * from './youtube-dl-cli' | ||
2 | export * from './youtube-dl-info-builder' | ||
3 | export * from './youtube-dl-wrapper' | ||
diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts deleted file mode 100644 index 765038cea..000000000 --- a/server/helpers/youtube-dl/youtube-dl-cli.ts +++ /dev/null | |||
@@ -1,259 +0,0 @@ | |||
1 | import execa from 'execa' | ||
2 | import { ensureDir, pathExists, writeFile } from 'fs-extra' | ||
3 | import { dirname, join } from 'path' | ||
4 | import { CONFIG } from '@server/initializers/config' | ||
5 | import { VideoResolution } from '@shared/models' | ||
6 | import { logger, loggerTagsFactory } from '../logger' | ||
7 | import { getProxy, isProxyEnabled } from '../proxy' | ||
8 | import { isBinaryResponse, peertubeGot } from '../requests' | ||
9 | import { OptionsOfBufferResponseBody } from 'got/dist/source' | ||
10 | |||
11 | const lTags = loggerTagsFactory('youtube-dl') | ||
12 | |||
13 | const youtubeDLBinaryPath = join(CONFIG.STORAGE.BIN_DIR, CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME) | ||
14 | |||
15 | export class YoutubeDLCLI { | ||
16 | |||
17 | static async safeGet () { | ||
18 | if (!await pathExists(youtubeDLBinaryPath)) { | ||
19 | await ensureDir(dirname(youtubeDLBinaryPath)) | ||
20 | |||
21 | await this.updateYoutubeDLBinary() | ||
22 | } | ||
23 | |||
24 | return new YoutubeDLCLI() | ||
25 | } | ||
26 | |||
27 | static async updateYoutubeDLBinary () { | ||
28 | const url = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.URL | ||
29 | |||
30 | logger.info('Updating youtubeDL binary from %s.', url, lTags()) | ||
31 | |||
32 | const gotOptions: OptionsOfBufferResponseBody = { | ||
33 | context: { bodyKBLimit: 20_000 }, | ||
34 | responseType: 'buffer' as 'buffer' | ||
35 | } | ||
36 | |||
37 | if (process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN) { | ||
38 | gotOptions.headers = { | ||
39 | authorization: 'Bearer ' + process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN | ||
40 | } | ||
41 | } | ||
42 | |||
43 | try { | ||
44 | let gotResult = await peertubeGot(url, gotOptions) | ||
45 | |||
46 | if (!isBinaryResponse(gotResult)) { | ||
47 | const json = JSON.parse(gotResult.body.toString()) | ||
48 | const latest = json.filter(release => release.prerelease === false)[0] | ||
49 | if (!latest) throw new Error('Cannot find latest release') | ||
50 | |||
51 | const releaseName = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME | ||
52 | const releaseAsset = latest.assets.find(a => a.name === releaseName) | ||
53 | if (!releaseAsset) throw new Error(`Cannot find appropriate release with name ${releaseName} in release assets`) | ||
54 | |||
55 | gotResult = await peertubeGot(releaseAsset.browser_download_url, gotOptions) | ||
56 | } | ||
57 | |||
58 | if (!isBinaryResponse(gotResult)) { | ||
59 | throw new Error('Not a binary response') | ||
60 | } | ||
61 | |||
62 | await writeFile(youtubeDLBinaryPath, gotResult.body) | ||
63 | |||
64 | logger.info('youtube-dl updated %s.', youtubeDLBinaryPath, lTags()) | ||
65 | } catch (err) { | ||
66 | logger.error('Cannot update youtube-dl from %s.', url, { err, ...lTags() }) | ||
67 | } | ||
68 | } | ||
69 | |||
70 | static getYoutubeDLVideoFormat (enabledResolutions: VideoResolution[], useBestFormat: boolean) { | ||
71 | /** | ||
72 | * list of format selectors in order or preference | ||
73 | * see https://github.com/ytdl-org/youtube-dl#format-selection | ||
74 | * | ||
75 | * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope | ||
76 | * of being able to do a "quick-transcode" | ||
77 | * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9) | ||
78 | * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback | ||
79 | * | ||
80 | * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499 | ||
81 | **/ | ||
82 | |||
83 | let result: string[] = [] | ||
84 | |||
85 | if (!useBestFormat) { | ||
86 | const resolution = enabledResolutions.length === 0 | ||
87 | ? VideoResolution.H_720P | ||
88 | : Math.max(...enabledResolutions) | ||
89 | |||
90 | result = [ | ||
91 | `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1 | ||
92 | `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2 | ||
93 | `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]` // case # | ||
94 | ] | ||
95 | } | ||
96 | |||
97 | return result.concat([ | ||
98 | 'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio', | ||
99 | 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats | ||
100 | 'bestvideo[ext=mp4]+bestaudio[ext=m4a]', | ||
101 | 'best' // Ultimate fallback | ||
102 | ]).join('/') | ||
103 | } | ||
104 | |||
105 | private constructor () { | ||
106 | |||
107 | } | ||
108 | |||
109 | download (options: { | ||
110 | url: string | ||
111 | format: string | ||
112 | output: string | ||
113 | processOptions: execa.NodeOptions | ||
114 | timeout?: number | ||
115 | additionalYoutubeDLArgs?: string[] | ||
116 | }) { | ||
117 | let args = options.additionalYoutubeDLArgs || [] | ||
118 | args = args.concat([ '--merge-output-format', 'mp4', '-f', options.format, '-o', options.output ]) | ||
119 | |||
120 | return this.run({ | ||
121 | url: options.url, | ||
122 | processOptions: options.processOptions, | ||
123 | timeout: options.timeout, | ||
124 | args | ||
125 | }) | ||
126 | } | ||
127 | |||
128 | async getInfo (options: { | ||
129 | url: string | ||
130 | format: string | ||
131 | processOptions: execa.NodeOptions | ||
132 | additionalYoutubeDLArgs?: string[] | ||
133 | }) { | ||
134 | const { url, format, additionalYoutubeDLArgs = [], processOptions } = options | ||
135 | |||
136 | const completeArgs = additionalYoutubeDLArgs.concat([ '--dump-json', '-f', format ]) | ||
137 | |||
138 | const data = await this.run({ url, args: completeArgs, processOptions }) | ||
139 | if (!data) return undefined | ||
140 | |||
141 | const info = data.map(d => JSON.parse(d)) | ||
142 | |||
143 | return info.length === 1 | ||
144 | ? info[0] | ||
145 | : info | ||
146 | } | ||
147 | |||
148 | async getListInfo (options: { | ||
149 | url: string | ||
150 | latestVideosCount?: number | ||
151 | processOptions: execa.NodeOptions | ||
152 | }): Promise<{ upload_date: string, webpage_url: string }[]> { | ||
153 | const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ] | ||
154 | |||
155 | if (CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME === 'yt-dlp') { | ||
156 | // Optimize listing videos only when using yt-dlp because it is bugged with youtube-dl when fetching a channel | ||
157 | additionalYoutubeDLArgs.push('--flat-playlist') | ||
158 | } | ||
159 | |||
160 | if (options.latestVideosCount !== undefined) { | ||
161 | additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString()) | ||
162 | } | ||
163 | |||
164 | const result = await this.getInfo({ | ||
165 | url: options.url, | ||
166 | format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false), | ||
167 | processOptions: options.processOptions, | ||
168 | additionalYoutubeDLArgs | ||
169 | }) | ||
170 | |||
171 | if (!result) return result | ||
172 | if (!Array.isArray(result)) return [ result ] | ||
173 | |||
174 | return result | ||
175 | } | ||
176 | |||
177 | async getSubs (options: { | ||
178 | url: string | ||
179 | format: 'vtt' | ||
180 | processOptions: execa.NodeOptions | ||
181 | }) { | ||
182 | const { url, format, processOptions } = options | ||
183 | |||
184 | const args = [ '--skip-download', '--all-subs', `--sub-format=${format}` ] | ||
185 | |||
186 | const data = await this.run({ url, args, processOptions }) | ||
187 | const files: string[] = [] | ||
188 | |||
189 | const skipString = '[info] Writing video subtitles to: ' | ||
190 | |||
191 | for (let i = 0, len = data.length; i < len; i++) { | ||
192 | const line = data[i] | ||
193 | |||
194 | if (line.indexOf(skipString) === 0) { | ||
195 | files.push(line.slice(skipString.length)) | ||
196 | } | ||
197 | } | ||
198 | |||
199 | return files | ||
200 | } | ||
201 | |||
202 | private async run (options: { | ||
203 | url: string | ||
204 | args: string[] | ||
205 | timeout?: number | ||
206 | processOptions: execa.NodeOptions | ||
207 | }) { | ||
208 | const { url, args, timeout, processOptions } = options | ||
209 | |||
210 | let completeArgs = this.wrapWithProxyOptions(args) | ||
211 | completeArgs = this.wrapWithIPOptions(completeArgs) | ||
212 | completeArgs = this.wrapWithFFmpegOptions(completeArgs) | ||
213 | |||
214 | const { PYTHON_PATH } = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE | ||
215 | const subProcess = execa(PYTHON_PATH, [ youtubeDLBinaryPath, ...completeArgs, url ], processOptions) | ||
216 | |||
217 | if (timeout) { | ||
218 | setTimeout(() => subProcess.cancel(), timeout) | ||
219 | } | ||
220 | |||
221 | const output = await subProcess | ||
222 | |||
223 | logger.debug('Run youtube-dl command.', { command: output.command, ...lTags() }) | ||
224 | |||
225 | return output.stdout | ||
226 | ? output.stdout.trim().split(/\r?\n/) | ||
227 | : undefined | ||
228 | } | ||
229 | |||
230 | private wrapWithProxyOptions (args: string[]) { | ||
231 | if (isProxyEnabled()) { | ||
232 | logger.debug('Using proxy %s for YoutubeDL', getProxy(), lTags()) | ||
233 | |||
234 | return [ '--proxy', getProxy() ].concat(args) | ||
235 | } | ||
236 | |||
237 | return args | ||
238 | } | ||
239 | |||
240 | private wrapWithIPOptions (args: string[]) { | ||
241 | if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) { | ||
242 | logger.debug('Force ipv4 for YoutubeDL') | ||
243 | |||
244 | return [ '--force-ipv4' ].concat(args) | ||
245 | } | ||
246 | |||
247 | return args | ||
248 | } | ||
249 | |||
250 | private wrapWithFFmpegOptions (args: string[]) { | ||
251 | if (process.env.FFMPEG_PATH) { | ||
252 | logger.debug('Using ffmpeg location %s for YoutubeDL', process.env.FFMPEG_PATH, lTags()) | ||
253 | |||
254 | return [ '--ffmpeg-location', process.env.FFMPEG_PATH ].concat(args) | ||
255 | } | ||
256 | |||
257 | return args | ||
258 | } | ||
259 | } | ||
diff --git a/server/helpers/youtube-dl/youtube-dl-info-builder.ts b/server/helpers/youtube-dl/youtube-dl-info-builder.ts deleted file mode 100644 index a74904e43..000000000 --- a/server/helpers/youtube-dl/youtube-dl-info-builder.ts +++ /dev/null | |||
@@ -1,205 +0,0 @@ | |||
1 | import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../../initializers/constants' | ||
2 | import { peertubeTruncate } from '../core-utils' | ||
3 | import { isUrlValid } from '../custom-validators/activitypub/misc' | ||
4 | |||
5 | type YoutubeDLInfo = { | ||
6 | name?: string | ||
7 | description?: string | ||
8 | category?: number | ||
9 | language?: string | ||
10 | licence?: number | ||
11 | nsfw?: boolean | ||
12 | tags?: string[] | ||
13 | thumbnailUrl?: string | ||
14 | ext?: string | ||
15 | originallyPublishedAtWithoutTime?: Date | ||
16 | webpageUrl?: string | ||
17 | |||
18 | urls?: string[] | ||
19 | } | ||
20 | |||
21 | class YoutubeDLInfoBuilder { | ||
22 | private readonly info: any | ||
23 | |||
24 | constructor (info: any) { | ||
25 | this.info = { ...info } | ||
26 | } | ||
27 | |||
28 | getInfo () { | ||
29 | const obj = this.buildVideoInfo(this.normalizeObject(this.info)) | ||
30 | if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video' | ||
31 | |||
32 | return obj | ||
33 | } | ||
34 | |||
35 | private normalizeObject (obj: any) { | ||
36 | const newObj: any = {} | ||
37 | |||
38 | for (const key of Object.keys(obj)) { | ||
39 | // Deprecated key | ||
40 | if (key === 'resolution') continue | ||
41 | |||
42 | const value = obj[key] | ||
43 | |||
44 | if (typeof value === 'string') { | ||
45 | newObj[key] = value.normalize() | ||
46 | } else { | ||
47 | newObj[key] = value | ||
48 | } | ||
49 | } | ||
50 | |||
51 | return newObj | ||
52 | } | ||
53 | |||
54 | private buildOriginallyPublishedAt (obj: any) { | ||
55 | let originallyPublishedAt: Date = null | ||
56 | |||
57 | const uploadDateMatcher = /^(\d{4})(\d{2})(\d{2})$/.exec(obj.upload_date) | ||
58 | if (uploadDateMatcher) { | ||
59 | originallyPublishedAt = new Date() | ||
60 | originallyPublishedAt.setHours(0, 0, 0, 0) | ||
61 | |||
62 | const year = parseInt(uploadDateMatcher[1], 10) | ||
63 | // Month starts from 0 | ||
64 | const month = parseInt(uploadDateMatcher[2], 10) - 1 | ||
65 | const day = parseInt(uploadDateMatcher[3], 10) | ||
66 | |||
67 | originallyPublishedAt.setFullYear(year, month, day) | ||
68 | } | ||
69 | |||
70 | return originallyPublishedAt | ||
71 | } | ||
72 | |||
73 | private buildVideoInfo (obj: any): YoutubeDLInfo { | ||
74 | return { | ||
75 | name: this.titleTruncation(obj.title), | ||
76 | description: this.descriptionTruncation(obj.description), | ||
77 | category: this.getCategory(obj.categories), | ||
78 | licence: this.getLicence(obj.license), | ||
79 | language: this.getLanguage(obj.language), | ||
80 | nsfw: this.isNSFW(obj), | ||
81 | tags: this.getTags(obj.tags), | ||
82 | thumbnailUrl: obj.thumbnail || undefined, | ||
83 | urls: this.buildAvailableUrl(obj), | ||
84 | originallyPublishedAtWithoutTime: this.buildOriginallyPublishedAt(obj), | ||
85 | ext: obj.ext, | ||
86 | webpageUrl: obj.webpage_url | ||
87 | } | ||
88 | } | ||
89 | |||
90 | private buildAvailableUrl (obj: any) { | ||
91 | const urls: string[] = [] | ||
92 | |||
93 | if (obj.url) urls.push(obj.url) | ||
94 | if (obj.urls) { | ||
95 | if (Array.isArray(obj.urls)) urls.push(...obj.urls) | ||
96 | else urls.push(obj.urls) | ||
97 | } | ||
98 | |||
99 | const formats = Array.isArray(obj.formats) | ||
100 | ? obj.formats | ||
101 | : [] | ||
102 | |||
103 | for (const format of formats) { | ||
104 | if (!format.url) continue | ||
105 | |||
106 | urls.push(format.url) | ||
107 | } | ||
108 | |||
109 | const thumbnails = Array.isArray(obj.thumbnails) | ||
110 | ? obj.thumbnails | ||
111 | : [] | ||
112 | |||
113 | for (const thumbnail of thumbnails) { | ||
114 | if (!thumbnail.url) continue | ||
115 | |||
116 | urls.push(thumbnail.url) | ||
117 | } | ||
118 | |||
119 | if (obj.thumbnail) urls.push(obj.thumbnail) | ||
120 | |||
121 | for (const subtitleKey of Object.keys(obj.subtitles || {})) { | ||
122 | const subtitles = obj.subtitles[subtitleKey] | ||
123 | if (!Array.isArray(subtitles)) continue | ||
124 | |||
125 | for (const subtitle of subtitles) { | ||
126 | if (!subtitle.url) continue | ||
127 | |||
128 | urls.push(subtitle.url) | ||
129 | } | ||
130 | } | ||
131 | |||
132 | return urls.filter(u => u && isUrlValid(u)) | ||
133 | } | ||
134 | |||
135 | private titleTruncation (title: string) { | ||
136 | return peertubeTruncate(title, { | ||
137 | length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max, | ||
138 | separator: /,? +/, | ||
139 | omission: ' […]' | ||
140 | }) | ||
141 | } | ||
142 | |||
143 | private descriptionTruncation (description: string) { | ||
144 | if (!description || description.length < CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.min) return undefined | ||
145 | |||
146 | return peertubeTruncate(description, { | ||
147 | length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max, | ||
148 | separator: /,? +/, | ||
149 | omission: ' […]' | ||
150 | }) | ||
151 | } | ||
152 | |||
153 | private isNSFW (info: any) { | ||
154 | return info?.age_limit >= 16 | ||
155 | } | ||
156 | |||
157 | private getTags (tags: string[]) { | ||
158 | if (Array.isArray(tags) === false) return [] | ||
159 | |||
160 | return tags | ||
161 | .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min) | ||
162 | .map(t => t.normalize()) | ||
163 | .slice(0, 5) | ||
164 | } | ||
165 | |||
166 | private getLicence (licence: string) { | ||
167 | if (!licence) return undefined | ||
168 | |||
169 | if (licence.includes('Creative Commons Attribution')) return 1 | ||
170 | |||
171 | for (const key of Object.keys(VIDEO_LICENCES)) { | ||
172 | const peertubeLicence = VIDEO_LICENCES[key] | ||
173 | if (peertubeLicence.toLowerCase() === licence.toLowerCase()) return parseInt(key, 10) | ||
174 | } | ||
175 | |||
176 | return undefined | ||
177 | } | ||
178 | |||
179 | private getCategory (categories: string[]) { | ||
180 | if (!categories) return undefined | ||
181 | |||
182 | const categoryString = categories[0] | ||
183 | if (!categoryString || typeof categoryString !== 'string') return undefined | ||
184 | |||
185 | if (categoryString === 'News & Politics') return 11 | ||
186 | |||
187 | for (const key of Object.keys(VIDEO_CATEGORIES)) { | ||
188 | const category = VIDEO_CATEGORIES[key] | ||
189 | if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10) | ||
190 | } | ||
191 | |||
192 | return undefined | ||
193 | } | ||
194 | |||
195 | private getLanguage (language: string) { | ||
196 | return VIDEO_LANGUAGES[language] ? language : undefined | ||
197 | } | ||
198 | } | ||
199 | |||
200 | // --------------------------------------------------------------------------- | ||
201 | |||
202 | export { | ||
203 | YoutubeDLInfo, | ||
204 | YoutubeDLInfoBuilder | ||
205 | } | ||
diff --git a/server/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/helpers/youtube-dl/youtube-dl-wrapper.ts deleted file mode 100644 index ac3cd190e..000000000 --- a/server/helpers/youtube-dl/youtube-dl-wrapper.ts +++ /dev/null | |||
@@ -1,154 +0,0 @@ | |||
1 | import { move, pathExists, readdir, remove } from 'fs-extra' | ||
2 | import { dirname, join } from 'path' | ||
3 | import { inspect } from 'util' | ||
4 | import { CONFIG } from '@server/initializers/config' | ||
5 | import { isVideoFileExtnameValid } from '../custom-validators/videos' | ||
6 | import { logger, loggerTagsFactory } from '../logger' | ||
7 | import { generateVideoImportTmpPath } from '../utils' | ||
8 | import { YoutubeDLCLI } from './youtube-dl-cli' | ||
9 | import { YoutubeDLInfo, YoutubeDLInfoBuilder } from './youtube-dl-info-builder' | ||
10 | |||
11 | const lTags = loggerTagsFactory('youtube-dl') | ||
12 | |||
13 | export type YoutubeDLSubs = { | ||
14 | language: string | ||
15 | filename: string | ||
16 | path: string | ||
17 | }[] | ||
18 | |||
19 | const processOptions = { | ||
20 | maxBuffer: 1024 * 1024 * 30 // 30MB | ||
21 | } | ||
22 | |||
23 | class YoutubeDLWrapper { | ||
24 | |||
25 | constructor ( | ||
26 | private readonly url: string, | ||
27 | private readonly enabledResolutions: number[], | ||
28 | private readonly useBestFormat: boolean | ||
29 | ) { | ||
30 | |||
31 | } | ||
32 | |||
33 | async getInfoForDownload (youtubeDLArgs: string[] = []): Promise<YoutubeDLInfo> { | ||
34 | const youtubeDL = await YoutubeDLCLI.safeGet() | ||
35 | |||
36 | const info = await youtubeDL.getInfo({ | ||
37 | url: this.url, | ||
38 | format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat), | ||
39 | additionalYoutubeDLArgs: youtubeDLArgs, | ||
40 | processOptions | ||
41 | }) | ||
42 | |||
43 | if (!info) throw new Error(`YoutubeDL could not get info from ${this.url}`) | ||
44 | |||
45 | if (info.is_live === true) throw new Error('Cannot download a live streaming.') | ||
46 | |||
47 | const infoBuilder = new YoutubeDLInfoBuilder(info) | ||
48 | |||
49 | return infoBuilder.getInfo() | ||
50 | } | ||
51 | |||
52 | async getInfoForListImport (options: { | ||
53 | latestVideosCount?: number | ||
54 | }) { | ||
55 | const youtubeDL = await YoutubeDLCLI.safeGet() | ||
56 | |||
57 | const list = await youtubeDL.getListInfo({ | ||
58 | url: this.url, | ||
59 | latestVideosCount: options.latestVideosCount, | ||
60 | processOptions | ||
61 | }) | ||
62 | |||
63 | if (!Array.isArray(list)) throw new Error(`YoutubeDL could not get list info from ${this.url}: ${inspect(list)}`) | ||
64 | |||
65 | return list.map(info => info.webpage_url) | ||
66 | } | ||
67 | |||
68 | async getSubtitles (): Promise<YoutubeDLSubs> { | ||
69 | const cwd = CONFIG.STORAGE.TMP_DIR | ||
70 | |||
71 | const youtubeDL = await YoutubeDLCLI.safeGet() | ||
72 | |||
73 | const files = await youtubeDL.getSubs({ url: this.url, format: 'vtt', processOptions: { cwd } }) | ||
74 | if (!files) return [] | ||
75 | |||
76 | logger.debug('Get subtitles from youtube dl.', { url: this.url, files, ...lTags() }) | ||
77 | |||
78 | const subtitles = files.reduce((acc, filename) => { | ||
79 | const matched = filename.match(/\.([a-z]{2})(-[a-z]+)?\.(vtt|ttml)/i) | ||
80 | if (!matched?.[1]) return acc | ||
81 | |||
82 | return [ | ||
83 | ...acc, | ||
84 | { | ||
85 | language: matched[1], | ||
86 | path: join(cwd, filename), | ||
87 | filename | ||
88 | } | ||
89 | ] | ||
90 | }, []) | ||
91 | |||
92 | return subtitles | ||
93 | } | ||
94 | |||
95 | async downloadVideo (fileExt: string, timeout: number): Promise<string> { | ||
96 | // Leave empty the extension, youtube-dl will add it | ||
97 | const pathWithoutExtension = generateVideoImportTmpPath(this.url, '') | ||
98 | |||
99 | logger.info('Importing youtubeDL video %s to %s', this.url, pathWithoutExtension, lTags()) | ||
100 | |||
101 | const youtubeDL = await YoutubeDLCLI.safeGet() | ||
102 | |||
103 | try { | ||
104 | await youtubeDL.download({ | ||
105 | url: this.url, | ||
106 | format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat), | ||
107 | output: pathWithoutExtension, | ||
108 | timeout, | ||
109 | processOptions | ||
110 | }) | ||
111 | |||
112 | // If youtube-dl did not guess an extension for our file, just use .mp4 as default | ||
113 | if (await pathExists(pathWithoutExtension)) { | ||
114 | await move(pathWithoutExtension, pathWithoutExtension + '.mp4') | ||
115 | } | ||
116 | |||
117 | return this.guessVideoPathWithExtension(pathWithoutExtension, fileExt) | ||
118 | } catch (err) { | ||
119 | this.guessVideoPathWithExtension(pathWithoutExtension, fileExt) | ||
120 | .then(path => { | ||
121 | logger.debug('Error in youtube-dl import, deleting file %s.', path, { err, ...lTags() }) | ||
122 | |||
123 | return remove(path) | ||
124 | }) | ||
125 | .catch(innerErr => logger.error('Cannot remove file in youtubeDL error.', { innerErr, ...lTags() })) | ||
126 | |||
127 | throw err | ||
128 | } | ||
129 | } | ||
130 | |||
131 | private async guessVideoPathWithExtension (tmpPath: string, sourceExt: string) { | ||
132 | if (!isVideoFileExtnameValid(sourceExt)) { | ||
133 | throw new Error('Invalid video extension ' + sourceExt) | ||
134 | } | ||
135 | |||
136 | const extensions = [ sourceExt, '.mp4', '.mkv', '.webm' ] | ||
137 | |||
138 | for (const extension of extensions) { | ||
139 | const path = tmpPath + extension | ||
140 | |||
141 | if (await pathExists(path)) return path | ||
142 | } | ||
143 | |||
144 | const directoryContent = await readdir(dirname(tmpPath)) | ||
145 | |||
146 | throw new Error(`Cannot guess path of ${tmpPath}. Directory content: ${directoryContent.join(', ')}`) | ||
147 | } | ||
148 | } | ||
149 | |||
150 | // --------------------------------------------------------------------------- | ||
151 | |||
152 | export { | ||
153 | YoutubeDLWrapper | ||
154 | } | ||