aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/helpers
diff options
context:
space:
mode:
Diffstat (limited to 'server/helpers')
-rw-r--r--server/helpers/activitypub.ts81
-rw-r--r--server/helpers/audit-logger.ts24
-rw-r--r--server/helpers/core-utils.ts68
-rw-r--r--server/helpers/custom-jsonld-signature.ts90
-rw-r--r--server/helpers/custom-validators/activitypub/actor.ts4
-rw-r--r--server/helpers/custom-validators/activitypub/cache-file.ts2
-rw-r--r--server/helpers/custom-validators/activitypub/video-comments.ts2
-rw-r--r--server/helpers/custom-validators/activitypub/videos.ts27
-rw-r--r--server/helpers/custom-validators/misc.ts4
-rw-r--r--server/helpers/custom-validators/plugins.ts6
-rw-r--r--server/helpers/custom-validators/user-notifications.ts3
-rw-r--r--server/helpers/custom-validators/users.ts9
-rw-r--r--server/helpers/custom-validators/video-abuses.ts4
-rw-r--r--server/helpers/custom-validators/video-captions.ts2
-rw-r--r--server/helpers/custom-validators/video-imports.ts2
-rw-r--r--server/helpers/custom-validators/video-playlists.ts6
-rw-r--r--server/helpers/custom-validators/video-redundancies.ts12
-rw-r--r--server/helpers/custom-validators/videos.ts8
-rw-r--r--server/helpers/express-utils.ts16
-rw-r--r--server/helpers/ffmpeg-utils.ts191
-rw-r--r--server/helpers/logger.ts15
-rw-r--r--server/helpers/middlewares/videos.ts17
-rw-r--r--server/helpers/regexp.ts2
-rw-r--r--server/helpers/register-ts-paths.ts2
-rw-r--r--server/helpers/signup.ts2
-rw-r--r--server/helpers/utils.ts4
-rw-r--r--server/helpers/video.ts27
-rw-r--r--server/helpers/webtorrent.ts24
-rw-r--r--server/helpers/youtube-dl.ts61
29 files changed, 408 insertions, 307 deletions
diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts
index 239d8291d..2d49e6869 100644
--- a/server/helpers/activitypub.ts
+++ b/server/helpers/activitypub.ts
@@ -2,21 +2,36 @@ import * as Bluebird from 'bluebird'
2import validator from 'validator' 2import validator from 'validator'
3import { ResultList } from '../../shared/models' 3import { ResultList } from '../../shared/models'
4import { Activity } from '../../shared/models/activitypub' 4import { Activity } from '../../shared/models/activitypub'
5import { ACTIVITY_PUB } from '../initializers/constants' 5import { ACTIVITY_PUB, REMOTE_SCHEME } from '../initializers/constants'
6import { signJsonLDObject } from './peertube-crypto' 6import { signJsonLDObject } from './peertube-crypto'
7import { pageToStartAndCount } from './core-utils' 7import { pageToStartAndCount } from './core-utils'
8import { parse } from 'url' 8import { URL } from 'url'
9import { MActor } from '../typings/models' 9import { MActor, MVideoAccountLight } from '../typings/models'
10 10
11function activityPubContextify <T> (data: T) { 11export type ContextType = 'All' | 'View' | 'Announce' | 'CacheFile'
12 return Object.assign(data, { 12
13 '@context': [ 13function getContextData (type: ContextType) {
14 'https://www.w3.org/ns/activitystreams', 14 const context: any[] = [
15 'https://w3id.org/security/v1', 15 'https://www.w3.org/ns/activitystreams',
16 { 16 'https://w3id.org/security/v1',
17 RsaSignature2017: 'https://w3id.org/security#RsaSignature2017', 17 {
18 pt: 'https://joinpeertube.org/ns#', 18 RsaSignature2017: 'https://w3id.org/security#RsaSignature2017'
19 sc: 'http://schema.org#', 19 }
20 ]
21
22 if (type !== 'View' && type !== 'Announce') {
23 const additional = {
24 pt: 'https://joinpeertube.org/ns#',
25 sc: 'http://schema.org#'
26 }
27
28 if (type === 'CacheFile') {
29 Object.assign(additional, {
30 expires: 'sc:expires',
31 CacheFile: 'pt:CacheFile'
32 })
33 } else {
34 Object.assign(additional, {
20 Hashtag: 'as:Hashtag', 35 Hashtag: 'as:Hashtag',
21 uuid: 'sc:identifier', 36 uuid: 'sc:identifier',
22 category: 'sc:category', 37 category: 'sc:category',
@@ -24,8 +39,7 @@ function activityPubContextify <T> (data: T) {
24 subtitleLanguage: 'sc:subtitleLanguage', 39 subtitleLanguage: 'sc:subtitleLanguage',
25 sensitive: 'as:sensitive', 40 sensitive: 'as:sensitive',
26 language: 'sc:inLanguage', 41 language: 'sc:inLanguage',
27 expires: 'sc:expires', 42
28 CacheFile: 'pt:CacheFile',
29 Infohash: 'pt:Infohash', 43 Infohash: 'pt:Infohash',
30 originallyPublishedAt: 'sc:datePublished', 44 originallyPublishedAt: 'sc:datePublished',
31 views: { 45 views: {
@@ -71,9 +85,7 @@ function activityPubContextify <T> (data: T) {
71 support: { 85 support: {
72 '@type': 'sc:Text', 86 '@type': 'sc:Text',
73 '@id': 'pt:support' 87 '@id': 'pt:support'
74 } 88 },
75 },
76 {
77 likes: { 89 likes: {
78 '@id': 'as:likes', 90 '@id': 'as:likes',
79 '@type': '@id' 91 '@type': '@id'
@@ -94,9 +106,19 @@ function activityPubContextify <T> (data: T) {
94 '@id': 'as:comments', 106 '@id': 'as:comments',
95 '@type': '@id' 107 '@type': '@id'
96 } 108 }
97 } 109 })
98 ] 110 }
99 }) 111
112 context.push(additional)
113 }
114
115 return {
116 '@context': context
117 }
118}
119
120function activityPubContextify <T> (data: T, type: ContextType = 'All') {
121 return Object.assign({}, data, getContextData(type))
100} 122}
101 123
102type ActivityPubCollectionPaginationHandler = (start: number, count: number) => Bluebird<ResultList<any>> | Promise<ResultList<any>> 124type ActivityPubCollectionPaginationHandler = (start: number, count: number) => Bluebird<ResultList<any>> | Promise<ResultList<any>>
@@ -148,8 +170,8 @@ async function activityPubCollectionPagination (
148 170
149} 171}
150 172
151function buildSignedActivity (byActor: MActor, data: Object) { 173function buildSignedActivity (byActor: MActor, data: Object, contextType?: ContextType) {
152 const activity = activityPubContextify(data) 174 const activity = activityPubContextify(data, contextType)
153 175
154 return signJsonLDObject(byActor, activity) as Promise<Activity> 176 return signJsonLDObject(byActor, activity) as Promise<Activity>
155} 177}
@@ -161,12 +183,18 @@ function getAPId (activity: string | { id: string }) {
161} 183}
162 184
163function checkUrlsSameHost (url1: string, url2: string) { 185function checkUrlsSameHost (url1: string, url2: string) {
164 const idHost = parse(url1).host 186 const idHost = new URL(url1).host
165 const actorHost = parse(url2).host 187 const actorHost = new URL(url2).host
166 188
167 return idHost && actorHost && idHost.toLowerCase() === actorHost.toLowerCase() 189 return idHost && actorHost && idHost.toLowerCase() === actorHost.toLowerCase()
168} 190}
169 191
192function buildRemoteVideoBaseUrl (video: MVideoAccountLight, path: string) {
193 const host = video.VideoChannel.Account.Actor.Server.host
194
195 return REMOTE_SCHEME.HTTP + '://' + host + path
196}
197
170// --------------------------------------------------------------------------- 198// ---------------------------------------------------------------------------
171 199
172export { 200export {
@@ -174,5 +202,6 @@ export {
174 getAPId, 202 getAPId,
175 activityPubContextify, 203 activityPubContextify,
176 activityPubCollectionPagination, 204 activityPubCollectionPagination,
177 buildSignedActivity 205 buildSignedActivity,
206 buildRemoteVideoBaseUrl
178} 207}
diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts
index 9b258dc3a..a4cfeef76 100644
--- a/server/helpers/audit-logger.ts
+++ b/server/helpers/audit-logger.ts
@@ -81,7 +81,8 @@ function auditLoggerFactory (domain: string) {
81} 81}
82 82
83abstract class EntityAuditView { 83abstract class EntityAuditView {
84 constructor (private keysToKeep: Array<string>, private prefix: string, private entityInfos: object) { } 84 constructor (private readonly keysToKeep: string[], private readonly prefix: string, private readonly entityInfos: object) { }
85
85 toLogKeys (): object { 86 toLogKeys (): object {
86 return chain(flatten(this.entityInfos, { delimiter: '-', safe: true })) 87 return chain(flatten(this.entityInfos, { delimiter: '-', safe: true }))
87 .pick(this.keysToKeep) 88 .pick(this.keysToKeep)
@@ -121,7 +122,7 @@ const videoKeysToKeep = [
121 'downloadEnabled' 122 'downloadEnabled'
122] 123]
123class VideoAuditView extends EntityAuditView { 124class VideoAuditView extends EntityAuditView {
124 constructor (private video: VideoDetails) { 125 constructor (private readonly video: VideoDetails) {
125 super(videoKeysToKeep, 'video', video) 126 super(videoKeysToKeep, 'video', video)
126 } 127 }
127} 128}
@@ -132,7 +133,7 @@ const videoImportKeysToKeep = [
132 'video-name' 133 'video-name'
133] 134]
134class VideoImportAuditView extends EntityAuditView { 135class VideoImportAuditView extends EntityAuditView {
135 constructor (private videoImport: VideoImport) { 136 constructor (private readonly videoImport: VideoImport) {
136 super(videoImportKeysToKeep, 'video-import', videoImport) 137 super(videoImportKeysToKeep, 'video-import', videoImport)
137 } 138 }
138} 139}
@@ -151,7 +152,7 @@ const commentKeysToKeep = [
151 'account-name' 152 'account-name'
152] 153]
153class CommentAuditView extends EntityAuditView { 154class CommentAuditView extends EntityAuditView {
154 constructor (private comment: VideoComment) { 155 constructor (private readonly comment: VideoComment) {
155 super(commentKeysToKeep, 'comment', comment) 156 super(commentKeysToKeep, 'comment', comment)
156 } 157 }
157} 158}
@@ -180,7 +181,7 @@ const userKeysToKeep = [
180 'videoChannels' 181 'videoChannels'
181] 182]
182class UserAuditView extends EntityAuditView { 183class UserAuditView extends EntityAuditView {
183 constructor (private user: User) { 184 constructor (private readonly user: User) {
184 super(userKeysToKeep, 'user', user) 185 super(userKeysToKeep, 'user', user)
185 } 186 }
186} 187}
@@ -206,7 +207,7 @@ const channelKeysToKeep = [
206 'ownerAccount-displayedName' 207 'ownerAccount-displayedName'
207] 208]
208class VideoChannelAuditView extends EntityAuditView { 209class VideoChannelAuditView extends EntityAuditView {
209 constructor (private channel: VideoChannel) { 210 constructor (private readonly channel: VideoChannel) {
210 super(channelKeysToKeep, 'channel', channel) 211 super(channelKeysToKeep, 'channel', channel)
211 } 212 }
212} 213}
@@ -221,7 +222,7 @@ const videoAbuseKeysToKeep = [
221 'createdAt' 222 'createdAt'
222] 223]
223class VideoAbuseAuditView extends EntityAuditView { 224class VideoAbuseAuditView extends EntityAuditView {
224 constructor (private videoAbuse: VideoAbuse) { 225 constructor (private readonly videoAbuse: VideoAbuse) {
225 super(videoAbuseKeysToKeep, 'abuse', videoAbuse) 226 super(videoAbuseKeysToKeep, 'abuse', videoAbuse)
226 } 227 }
227} 228}
@@ -253,9 +254,12 @@ class CustomConfigAuditView extends EntityAuditView {
253 const infos: any = customConfig 254 const infos: any = customConfig
254 const resolutionsDict = infos.transcoding.resolutions 255 const resolutionsDict = infos.transcoding.resolutions
255 const resolutionsArray = [] 256 const resolutionsArray = []
256 Object.entries(resolutionsDict).forEach(([resolution, isEnabled]) => { 257
257 if (isEnabled) resolutionsArray.push(resolution) 258 Object.entries(resolutionsDict)
258 }) 259 .forEach(([ resolution, isEnabled ]) => {
260 if (isEnabled) resolutionsArray.push(resolution)
261 })
262
259 Object.assign({}, infos, { transcoding: { resolutions: resolutionsArray } }) 263 Object.assign({}, infos, { transcoding: { resolutions: resolutionsArray } })
260 super(customConfigKeysToKeep, 'config', infos) 264 super(customConfigKeysToKeep, 'config', infos)
261 } 265 }
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index 7e8252aa4..b1f5d9610 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -1,9 +1,11 @@
1/* eslint-disable no-useless-call */
2
1/* 3/*
2 Different from 'utils' because we don't not import other PeerTube modules. 4 Different from 'utils' because we don't import other PeerTube modules.
3 Useful to avoid circular dependencies. 5 Useful to avoid circular dependencies.
4*/ 6*/
5 7
6import { createHash, HexBase64Latin1Encoding, pseudoRandomBytes } from 'crypto' 8import { createHash, HexBase64Latin1Encoding, randomBytes } from 'crypto'
7import { basename, isAbsolute, join, resolve } from 'path' 9import { basename, isAbsolute, join, resolve } from 'path'
8import * as pem from 'pem' 10import * as pem from 'pem'
9import { URL } from 'url' 11import { URL } from 'url'
@@ -22,31 +24,31 @@ const objectConverter = (oldObject: any, keyConverter: (e: string) => string, va
22 const newObject = {} 24 const newObject = {}
23 Object.keys(oldObject).forEach(oldKey => { 25 Object.keys(oldObject).forEach(oldKey => {
24 const newKey = keyConverter(oldKey) 26 const newKey = keyConverter(oldKey)
25 newObject[ newKey ] = objectConverter(oldObject[ oldKey ], keyConverter, valueConverter) 27 newObject[newKey] = objectConverter(oldObject[oldKey], keyConverter, valueConverter)
26 }) 28 })
27 29
28 return newObject 30 return newObject
29} 31}
30 32
31const timeTable = { 33const timeTable = {
32 ms: 1, 34 ms: 1,
33 second: 1000, 35 second: 1000,
34 minute: 60000, 36 minute: 60000,
35 hour: 3600000, 37 hour: 3600000,
36 day: 3600000 * 24, 38 day: 3600000 * 24,
37 week: 3600000 * 24 * 7, 39 week: 3600000 * 24 * 7,
38 month: 3600000 * 24 * 30 40 month: 3600000 * 24 * 30
39} 41}
40 42
41export function parseDurationToMs (duration: number | string): number { 43export function parseDurationToMs (duration: number | string): number {
42 if (typeof duration === 'number') return duration 44 if (typeof duration === 'number') return duration
43 45
44 if (typeof duration === 'string') { 46 if (typeof duration === 'string') {
45 const split = duration.match(/^([\d\.,]+)\s?(\w+)$/) 47 const split = duration.match(/^([\d.,]+)\s?(\w+)$/)
46 48
47 if (split.length === 3) { 49 if (split.length === 3) {
48 const len = parseFloat(split[1]) 50 const len = parseFloat(split[1])
49 let unit = split[2].replace(/s$/i,'').toLowerCase() 51 let unit = split[2].replace(/s$/i, '').toLowerCase()
50 if (unit === 'm') { 52 if (unit === 'm') {
51 unit = 'ms' 53 unit = 'ms'
52 } 54 }
@@ -73,21 +75,21 @@ export function parseBytes (value: string | number): number {
73 75
74 if (value.match(tgm)) { 76 if (value.match(tgm)) {
75 match = value.match(tgm) 77 match = value.match(tgm)
76 return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 78 return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 +
77 + parseInt(match[2], 10) * 1024 * 1024 * 1024 79 parseInt(match[2], 10) * 1024 * 1024 * 1024 +
78 + parseInt(match[3], 10) * 1024 * 1024 80 parseInt(match[3], 10) * 1024 * 1024
79 } else if (value.match(tg)) { 81 } else if (value.match(tg)) {
80 match = value.match(tg) 82 match = value.match(tg)
81 return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 83 return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 +
82 + parseInt(match[2], 10) * 1024 * 1024 * 1024 84 parseInt(match[2], 10) * 1024 * 1024 * 1024
83 } else if (value.match(tm)) { 85 } else if (value.match(tm)) {
84 match = value.match(tm) 86 match = value.match(tm)
85 return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 87 return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 +
86 + parseInt(match[2], 10) * 1024 * 1024 88 parseInt(match[2], 10) * 1024 * 1024
87 } else if (value.match(gm)) { 89 } else if (value.match(gm)) {
88 match = value.match(gm) 90 match = value.match(gm)
89 return parseInt(match[1], 10) * 1024 * 1024 * 1024 91 return parseInt(match[1], 10) * 1024 * 1024 * 1024 +
90 + parseInt(match[2], 10) * 1024 * 1024 92 parseInt(match[2], 10) * 1024 * 1024
91 } else if (value.match(t)) { 93 } else if (value.match(t)) {
92 match = value.match(t) 94 match = value.match(t)
93 return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 95 return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024
@@ -137,6 +139,7 @@ function getAppNumber () {
137} 139}
138 140
139let rootPath: string 141let rootPath: string
142
140function root () { 143function root () {
141 if (rootPath) return rootPath 144 if (rootPath) return rootPath
142 145
@@ -163,7 +166,7 @@ function escapeHTML (stringParam) {
163 '=': '&#x3D;' 166 '=': '&#x3D;'
164 } 167 }
165 168
166 return String(stringParam).replace(/[&<>"'`=\/]/g, s => entityMap[s]) 169 return String(stringParam).replace(/[&<>"'`=/]/g, s => entityMap[s])
167} 170}
168 171
169function pageToStartAndCount (page: number, itemsPerPage: number) { 172function pageToStartAndCount (page: number, itemsPerPage: number) {
@@ -202,6 +205,7 @@ function sha1 (str: string | Buffer, encoding: HexBase64Latin1Encoding = 'hex')
202function execShell (command: string, options?: ExecOptions) { 205function execShell (command: string, options?: ExecOptions) {
203 return new Promise<{ err?: Error, stdout: string, stderr: string }>((res, rej) => { 206 return new Promise<{ err?: Error, stdout: string, stderr: string }>((res, rej) => {
204 exec(command, options, (err, stdout, stderr) => { 207 exec(command, options, (err, stdout, stderr) => {
208 // eslint-disable-next-line prefer-promise-reject-errors
205 if (err) return rej({ err, stdout, stderr }) 209 if (err) return rej({ err, stdout, stderr })
206 210
207 return res({ stdout, stderr }) 211 return res({ stdout, stderr })
@@ -226,14 +230,6 @@ function promisify1<T, A> (func: (arg: T, cb: (err: any, result: A) => void) =>
226 } 230 }
227} 231}
228 232
229function promisify1WithVoid<T> (func: (arg: T, cb: (err: any) => void) => void): (arg: T) => Promise<void> {
230 return function promisified (arg: T): Promise<void> {
231 return new Promise<void>((resolve: () => void, reject: (err: any) => void) => {
232 func.apply(null, [ arg, (err: any) => err ? reject(err) : resolve() ])
233 })
234 }
235}
236
237function promisify2<T, U, A> (func: (arg1: T, arg2: U, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U) => Promise<A> { 233function promisify2<T, U, A> (func: (arg1: T, arg2: U, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U) => Promise<A> {
238 return function promisified (arg1: T, arg2: U): Promise<A> { 234 return function promisified (arg1: T, arg2: U): Promise<A> {
239 return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { 235 return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
@@ -242,15 +238,7 @@ function promisify2<T, U, A> (func: (arg1: T, arg2: U, cb: (err: any, result: A)
242 } 238 }
243} 239}
244 240
245function promisify2WithVoid<T, U> (func: (arg1: T, arg2: U, cb: (err: any) => void) => void): (arg1: T, arg2: U) => Promise<void> { 241const randomBytesPromise = promisify1<number, Buffer>(randomBytes)
246 return function promisified (arg1: T, arg2: U): Promise<void> {
247 return new Promise<void>((resolve: () => void, reject: (err: any) => void) => {
248 func.apply(null, [ arg1, arg2, (err: any) => err ? reject(err) : resolve() ])
249 })
250 }
251}
252
253const pseudoRandomBytesPromise = promisify1<number, Buffer>(pseudoRandomBytes)
254const createPrivateKey = promisify1<number, { key: string }>(pem.createPrivateKey) 242const createPrivateKey = promisify1<number, { key: string }>(pem.createPrivateKey)
255const getPublicKey = promisify1<string, { publicKey: string }>(pem.getPublicKey) 243const getPublicKey = promisify1<string, { publicKey: string }>(pem.getPublicKey)
256const execPromise2 = promisify2<string, any, string>(exec) 244const execPromise2 = promisify2<string, any, string>(exec)
@@ -280,7 +268,7 @@ export {
280 promisify1, 268 promisify1,
281 promisify2, 269 promisify2,
282 270
283 pseudoRandomBytesPromise, 271 randomBytesPromise,
284 createPrivateKey, 272 createPrivateKey,
285 getPublicKey, 273 getPublicKey,
286 execPromise2, 274 execPromise2,
diff --git a/server/helpers/custom-jsonld-signature.ts b/server/helpers/custom-jsonld-signature.ts
index a407a9fec..749c50cb3 100644
--- a/server/helpers/custom-jsonld-signature.ts
+++ b/server/helpers/custom-jsonld-signature.ts
@@ -5,52 +5,52 @@ import { logger } from './logger'
5const CACHE = { 5const CACHE = {
6 'https://w3id.org/security/v1': { 6 'https://w3id.org/security/v1': {
7 '@context': { 7 '@context': {
8 'id': '@id', 8 id: '@id',
9 'type': '@type', 9 type: '@type',
10 10
11 'dc': 'http://purl.org/dc/terms/', 11 dc: 'http://purl.org/dc/terms/',
12 'sec': 'https://w3id.org/security#', 12 sec: 'https://w3id.org/security#',
13 'xsd': 'http://www.w3.org/2001/XMLSchema#', 13 xsd: 'http://www.w3.org/2001/XMLSchema#',
14 14
15 'EcdsaKoblitzSignature2016': 'sec:EcdsaKoblitzSignature2016', 15 EcdsaKoblitzSignature2016: 'sec:EcdsaKoblitzSignature2016',
16 'Ed25519Signature2018': 'sec:Ed25519Signature2018', 16 Ed25519Signature2018: 'sec:Ed25519Signature2018',
17 'EncryptedMessage': 'sec:EncryptedMessage', 17 EncryptedMessage: 'sec:EncryptedMessage',
18 'GraphSignature2012': 'sec:GraphSignature2012', 18 GraphSignature2012: 'sec:GraphSignature2012',
19 'LinkedDataSignature2015': 'sec:LinkedDataSignature2015', 19 LinkedDataSignature2015: 'sec:LinkedDataSignature2015',
20 'LinkedDataSignature2016': 'sec:LinkedDataSignature2016', 20 LinkedDataSignature2016: 'sec:LinkedDataSignature2016',
21 'CryptographicKey': 'sec:Key', 21 CryptographicKey: 'sec:Key',
22 22
23 'authenticationTag': 'sec:authenticationTag', 23 authenticationTag: 'sec:authenticationTag',
24 'canonicalizationAlgorithm': 'sec:canonicalizationAlgorithm', 24 canonicalizationAlgorithm: 'sec:canonicalizationAlgorithm',
25 'cipherAlgorithm': 'sec:cipherAlgorithm', 25 cipherAlgorithm: 'sec:cipherAlgorithm',
26 'cipherData': 'sec:cipherData', 26 cipherData: 'sec:cipherData',
27 'cipherKey': 'sec:cipherKey', 27 cipherKey: 'sec:cipherKey',
28 'created': { '@id': 'dc:created', '@type': 'xsd:dateTime' }, 28 created: { '@id': 'dc:created', '@type': 'xsd:dateTime' },
29 'creator': { '@id': 'dc:creator', '@type': '@id' }, 29 creator: { '@id': 'dc:creator', '@type': '@id' },
30 'digestAlgorithm': 'sec:digestAlgorithm', 30 digestAlgorithm: 'sec:digestAlgorithm',
31 'digestValue': 'sec:digestValue', 31 digestValue: 'sec:digestValue',
32 'domain': 'sec:domain', 32 domain: 'sec:domain',
33 'encryptionKey': 'sec:encryptionKey', 33 encryptionKey: 'sec:encryptionKey',
34 'expiration': { '@id': 'sec:expiration', '@type': 'xsd:dateTime' }, 34 expiration: { '@id': 'sec:expiration', '@type': 'xsd:dateTime' },
35 'expires': { '@id': 'sec:expiration', '@type': 'xsd:dateTime' }, 35 expires: { '@id': 'sec:expiration', '@type': 'xsd:dateTime' },
36 'initializationVector': 'sec:initializationVector', 36 initializationVector: 'sec:initializationVector',
37 'iterationCount': 'sec:iterationCount', 37 iterationCount: 'sec:iterationCount',
38 'nonce': 'sec:nonce', 38 nonce: 'sec:nonce',
39 'normalizationAlgorithm': 'sec:normalizationAlgorithm', 39 normalizationAlgorithm: 'sec:normalizationAlgorithm',
40 'owner': { '@id': 'sec:owner', '@type': '@id' }, 40 owner: { '@id': 'sec:owner', '@type': '@id' },
41 'password': 'sec:password', 41 password: 'sec:password',
42 'privateKey': { '@id': 'sec:privateKey', '@type': '@id' }, 42 privateKey: { '@id': 'sec:privateKey', '@type': '@id' },
43 'privateKeyPem': 'sec:privateKeyPem', 43 privateKeyPem: 'sec:privateKeyPem',
44 'publicKey': { '@id': 'sec:publicKey', '@type': '@id' }, 44 publicKey: { '@id': 'sec:publicKey', '@type': '@id' },
45 'publicKeyBase58': 'sec:publicKeyBase58', 45 publicKeyBase58: 'sec:publicKeyBase58',
46 'publicKeyPem': 'sec:publicKeyPem', 46 publicKeyPem: 'sec:publicKeyPem',
47 'publicKeyWif': 'sec:publicKeyWif', 47 publicKeyWif: 'sec:publicKeyWif',
48 'publicKeyService': { '@id': 'sec:publicKeyService', '@type': '@id' }, 48 publicKeyService: { '@id': 'sec:publicKeyService', '@type': '@id' },
49 'revoked': { '@id': 'sec:revoked', '@type': 'xsd:dateTime' }, 49 revoked: { '@id': 'sec:revoked', '@type': 'xsd:dateTime' },
50 'salt': 'sec:salt', 50 salt: 'sec:salt',
51 'signature': 'sec:signature', 51 signature: 'sec:signature',
52 'signatureAlgorithm': 'sec:signingAlgorithm', 52 signatureAlgorithm: 'sec:signingAlgorithm',
53 'signatureValue': 'sec:signatureValue' 53 signatureValue: 'sec:signatureValue'
54 } 54 }
55 } 55 }
56} 56}
@@ -60,12 +60,12 @@ const nodeDocumentLoader = jsonld.documentLoaders.node()
60const lru = new AsyncLRU({ 60const lru = new AsyncLRU({
61 max: 10, 61 max: 10,
62 load: (url, cb) => { 62 load: (url, cb) => {
63 if (CACHE[ url ] !== undefined) { 63 if (CACHE[url] !== undefined) {
64 logger.debug('Using cache for JSON-LD %s.', url) 64 logger.debug('Using cache for JSON-LD %s.', url)
65 65
66 return cb(null, { 66 return cb(null, {
67 contextUrl: null, 67 contextUrl: null,
68 document: CACHE[ url ], 68 document: CACHE[url],
69 documentUrl: url 69 documentUrl: url
70 }) 70 })
71 } 71 }
diff --git a/server/helpers/custom-validators/activitypub/actor.ts b/server/helpers/custom-validators/activitypub/actor.ts
index fa58e163f..fec67823d 100644
--- a/server/helpers/custom-validators/activitypub/actor.ts
+++ b/server/helpers/custom-validators/activitypub/actor.ts
@@ -6,7 +6,7 @@ import { isHostValid } from '../servers'
6import { peertubeTruncate } from '@server/helpers/core-utils' 6import { peertubeTruncate } from '@server/helpers/core-utils'
7 7
8function isActorEndpointsObjectValid (endpointObject: any) { 8function isActorEndpointsObjectValid (endpointObject: any) {
9 if (endpointObject && endpointObject.sharedInbox) { 9 if (endpointObject?.sharedInbox) {
10 return isActivityPubUrlValid(endpointObject.sharedInbox) 10 return isActivityPubUrlValid(endpointObject.sharedInbox)
11 } 11 }
12 12
@@ -101,8 +101,6 @@ function normalizeActor (actor: any) {
101 actor.summary = null 101 actor.summary = null
102 } 102 }
103 } 103 }
104
105 return
106} 104}
107 105
108function isValidActorHandle (handle: string) { 106function isValidActorHandle (handle: string) {
diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts
index 21d5c53ca..c5b3b4d9f 100644
--- a/server/helpers/custom-validators/activitypub/cache-file.ts
+++ b/server/helpers/custom-validators/activitypub/cache-file.ts
@@ -6,7 +6,7 @@ import { CacheFileObject } from '../../../../shared/models/activitypub/objects'
6function isCacheFileObjectValid (object: CacheFileObject) { 6function isCacheFileObjectValid (object: CacheFileObject) {
7 return exists(object) && 7 return exists(object) &&
8 object.type === 'CacheFile' && 8 object.type === 'CacheFile' &&
9 isDateValid(object.expires) && 9 (object.expires === null || isDateValid(object.expires)) &&
10 isActivityPubUrlValid(object.object) && 10 isActivityPubUrlValid(object.object) &&
11 (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url)) 11 (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url))
12} 12}
diff --git a/server/helpers/custom-validators/activitypub/video-comments.ts b/server/helpers/custom-validators/activitypub/video-comments.ts
index aa3c246b5..ea852c491 100644
--- a/server/helpers/custom-validators/activitypub/video-comments.ts
+++ b/server/helpers/custom-validators/activitypub/video-comments.ts
@@ -48,8 +48,6 @@ function normalizeComment (comment: any) {
48 if (typeof comment.url === 'object') comment.url = comment.url.href || comment.url.url 48 if (typeof comment.url === 'object') comment.url = comment.url.href || comment.url.url
49 else comment.url = comment.id 49 else comment.url = comment.id
50 } 50 }
51
52 return
53} 51}
54 52
55function isCommentTypeValid (comment: any): boolean { 53function isCommentTypeValid (comment: any): boolean {
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index fe94bd58a..22b5e14a2 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -51,11 +51,16 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
51 logger.debug('Video has invalid captions', { video }) 51 logger.debug('Video has invalid captions', { video })
52 return false 52 return false
53 } 53 }
54 if (!setValidRemoteIcon(video)) {
55 logger.debug('Video has invalid icons', { video })
56 return false
57 }
54 58
55 // Default attributes 59 // Default attributes
56 if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED 60 if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED
57 if (!isBooleanValid(video.waitTranscoding)) video.waitTranscoding = false 61 if (!isBooleanValid(video.waitTranscoding)) video.waitTranscoding = false
58 if (!isBooleanValid(video.downloadEnabled)) video.downloadEnabled = true 62 if (!isBooleanValid(video.downloadEnabled)) video.downloadEnabled = true
63 if (!isBooleanValid(video.commentsEnabled)) video.commentsEnabled = false
59 64
60 return isActivityPubUrlValid(video.id) && 65 return isActivityPubUrlValid(video.id) &&
61 isVideoNameValid(video.name) && 66 isVideoNameValid(video.name) &&
@@ -72,7 +77,6 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
72 isDateValid(video.updated) && 77 isDateValid(video.updated) &&
73 (!video.originallyPublishedAt || isDateValid(video.originallyPublishedAt)) && 78 (!video.originallyPublishedAt || isDateValid(video.originallyPublishedAt)) &&
74 (!video.content || isRemoteVideoContentValid(video.mediaType, video.content)) && 79 (!video.content || isRemoteVideoContentValid(video.mediaType, video.content)) &&
75 isRemoteVideoIconValid(video.icon) &&
76 video.url.length !== 0 && 80 video.url.length !== 0 &&
77 video.attributedTo.length !== 0 81 video.attributedTo.length !== 0
78} 82}
@@ -131,6 +135,8 @@ function setValidRemoteCaptions (video: any) {
131 if (Array.isArray(video.subtitleLanguage) === false) return false 135 if (Array.isArray(video.subtitleLanguage) === false) return false
132 136
133 video.subtitleLanguage = video.subtitleLanguage.filter(caption => { 137 video.subtitleLanguage = video.subtitleLanguage.filter(caption => {
138 if (!isActivityPubUrlValid(caption.url)) caption.url = null
139
134 return isRemoteStringIdentifierValid(caption) 140 return isRemoteStringIdentifierValid(caption)
135 }) 141 })
136 142
@@ -149,12 +155,19 @@ function isRemoteVideoContentValid (mediaType: string, content: string) {
149 return mediaType === 'text/markdown' && isVideoTruncatedDescriptionValid(content) 155 return mediaType === 'text/markdown' && isVideoTruncatedDescriptionValid(content)
150} 156}
151 157
152function isRemoteVideoIconValid (icon: any) { 158function setValidRemoteIcon (video: any) {
153 return icon.type === 'Image' && 159 if (video.icon && !isArray(video.icon)) video.icon = [ video.icon ]
154 isActivityPubUrlValid(icon.url) && 160 if (!video.icon) video.icon = []
155 icon.mediaType === 'image/jpeg' && 161
156 validator.isInt(icon.width + '', { min: 0 }) && 162 video.icon = video.icon.filter(icon => {
157 validator.isInt(icon.height + '', { min: 0 }) 163 return icon.type === 'Image' &&
164 isActivityPubUrlValid(icon.url) &&
165 icon.mediaType === 'image/jpeg' &&
166 validator.isInt(icon.width + '', { min: 0 }) &&
167 validator.isInt(icon.height + '', { min: 0 })
168 })
169
170 return video.icon.length !== 0
158} 171}
159 172
160function setValidRemoteVideoUrls (video: any) { 173function setValidRemoteVideoUrls (video: any) {
diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts
index 89149b3e0..cf32201c4 100644
--- a/server/helpers/custom-validators/misc.ts
+++ b/server/helpers/custom-validators/misc.ts
@@ -94,13 +94,13 @@ function isFileValid (
94 if (isArray(files)) return optional 94 if (isArray(files)) return optional
95 95
96 // Should have a file 96 // Should have a file
97 const fileArray = files[ field ] 97 const fileArray = files[field]
98 if (!fileArray || fileArray.length === 0) { 98 if (!fileArray || fileArray.length === 0) {
99 return optional 99 return optional
100 } 100 }
101 101
102 // The file should exist 102 // The file should exist
103 const file = fileArray[ 0 ] 103 const file = fileArray[0]
104 if (!file || !file.originalname) return false 104 if (!file || !file.originalname) return false
105 105
106 // Check size 106 // Check size
diff --git a/server/helpers/custom-validators/plugins.ts b/server/helpers/custom-validators/plugins.ts
index 3af72547b..5a4531f72 100644
--- a/server/helpers/custom-validators/plugins.ts
+++ b/server/helpers/custom-validators/plugins.ts
@@ -14,7 +14,7 @@ function isPluginTypeValid (value: any) {
14function isPluginNameValid (value: string) { 14function isPluginNameValid (value: string) {
15 return exists(value) && 15 return exists(value) &&
16 validator.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.NAME) && 16 validator.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.NAME) &&
17 validator.matches(value, /^[a-z\-]+$/) 17 validator.matches(value, /^[a-z-]+$/)
18} 18}
19 19
20function isNpmPluginNameValid (value: string) { 20function isNpmPluginNameValid (value: string) {
@@ -146,8 +146,8 @@ function isPackageJSONValid (packageJSON: PluginPackageJson, pluginType: PluginT
146} 146}
147 147
148function isLibraryCodeValid (library: any) { 148function isLibraryCodeValid (library: any) {
149 return typeof library.register === 'function' 149 return typeof library.register === 'function' &&
150 && typeof library.unregister === 'function' 150 typeof library.unregister === 'function'
151} 151}
152 152
153export { 153export {
diff --git a/server/helpers/custom-validators/user-notifications.ts b/server/helpers/custom-validators/user-notifications.ts
index 5a4d10504..8a33b895b 100644
--- a/server/helpers/custom-validators/user-notifications.ts
+++ b/server/helpers/custom-validators/user-notifications.ts
@@ -9,7 +9,8 @@ function isUserNotificationTypeValid (value: any) {
9 9
10function isUserNotificationSettingValid (value: any) { 10function isUserNotificationSettingValid (value: any) {
11 return exists(value) && 11 return exists(value) &&
12 validator.isInt('' + value) && ( 12 validator.isInt('' + value) &&
13 (
13 value === UserNotificationSettingValue.NONE || 14 value === UserNotificationSettingValue.NONE ||
14 value === UserNotificationSettingValue.WEB || 15 value === UserNotificationSettingValue.WEB ||
15 value === UserNotificationSettingValue.EMAIL || 16 value === UserNotificationSettingValue.EMAIL ||
diff --git a/server/helpers/custom-validators/users.ts b/server/helpers/custom-validators/users.ts
index b4d5751e7..1ddbe0815 100644
--- a/server/helpers/custom-validators/users.ts
+++ b/server/helpers/custom-validators/users.ts
@@ -3,6 +3,7 @@ import { UserRole } from '../../../shared'
3import { CONSTRAINTS_FIELDS, NSFW_POLICY_TYPES } from '../../initializers/constants' 3import { CONSTRAINTS_FIELDS, NSFW_POLICY_TYPES } from '../../initializers/constants'
4import { exists, isArray, isBooleanValid, isFileValid } from './misc' 4import { exists, isArray, isBooleanValid, isFileValid } from './misc'
5import { values } from 'lodash' 5import { values } from 'lodash'
6import { isEmailEnabled } from '../../initializers/config'
6 7
7const USERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.USERS 8const USERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.USERS
8 9
@@ -10,6 +11,13 @@ function isUserPasswordValid (value: string) {
10 return validator.isLength(value, USERS_CONSTRAINTS_FIELDS.PASSWORD) 11 return validator.isLength(value, USERS_CONSTRAINTS_FIELDS.PASSWORD)
11} 12}
12 13
14function isUserPasswordValidOrEmpty (value: string) {
15 // Empty password is only possible if emailing is enabled.
16 if (value === '') return isEmailEnabled()
17
18 return isUserPasswordValid(value)
19}
20
13function isUserVideoQuotaValid (value: string) { 21function isUserVideoQuotaValid (value: string) {
14 return exists(value) && validator.isInt(value + '', USERS_CONSTRAINTS_FIELDS.VIDEO_QUOTA) 22 return exists(value) && validator.isInt(value + '', USERS_CONSTRAINTS_FIELDS.VIDEO_QUOTA)
15} 23}
@@ -103,6 +111,7 @@ export {
103 isUserVideosHistoryEnabledValid, 111 isUserVideosHistoryEnabledValid,
104 isUserBlockedValid, 112 isUserBlockedValid,
105 isUserPasswordValid, 113 isUserPasswordValid,
114 isUserPasswordValidOrEmpty,
106 isUserVideoLanguages, 115 isUserVideoLanguages,
107 isUserBlockedReasonValid, 116 isUserBlockedReasonValid,
108 isUserRoleValid, 117 isUserRoleValid,
diff --git a/server/helpers/custom-validators/video-abuses.ts b/server/helpers/custom-validators/video-abuses.ts
index a9478c76a..5c7bc6fd9 100644
--- a/server/helpers/custom-validators/video-abuses.ts
+++ b/server/helpers/custom-validators/video-abuses.ts
@@ -1,8 +1,6 @@
1import { Response } from 'express'
2import validator from 'validator' 1import validator from 'validator'
3import { CONSTRAINTS_FIELDS, VIDEO_ABUSE_STATES } from '../../initializers/constants' 2import { CONSTRAINTS_FIELDS, VIDEO_ABUSE_STATES } from '../../initializers/constants'
4import { exists } from './misc' 3import { exists } from './misc'
5import { VideoAbuseModel } from '../../models/video/video-abuse'
6 4
7const VIDEO_ABUSES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_ABUSES 5const VIDEO_ABUSES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_ABUSES
8 6
@@ -15,7 +13,7 @@ function isVideoAbuseModerationCommentValid (value: string) {
15} 13}
16 14
17function isVideoAbuseStateValid (value: string) { 15function isVideoAbuseStateValid (value: string) {
18 return exists(value) && VIDEO_ABUSE_STATES[ value ] !== undefined 16 return exists(value) && VIDEO_ABUSE_STATES[value] !== undefined
19} 17}
20 18
21// --------------------------------------------------------------------------- 19// ---------------------------------------------------------------------------
diff --git a/server/helpers/custom-validators/video-captions.ts b/server/helpers/custom-validators/video-captions.ts
index d06eb3695..9abbce04a 100644
--- a/server/helpers/custom-validators/video-captions.ts
+++ b/server/helpers/custom-validators/video-captions.ts
@@ -2,7 +2,7 @@ import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_LANGUAGES } from '../../initialize
2import { exists, isFileValid } from './misc' 2import { exists, isFileValid } from './misc'
3 3
4function isVideoCaptionLanguageValid (value: any) { 4function isVideoCaptionLanguageValid (value: any) {
5 return exists(value) && VIDEO_LANGUAGES[ value ] !== undefined 5 return exists(value) && VIDEO_LANGUAGES[value] !== undefined
6} 6}
7 7
8const videoCaptionTypes = Object.keys(MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT) 8const videoCaptionTypes = Object.keys(MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT)
diff --git a/server/helpers/custom-validators/video-imports.ts b/server/helpers/custom-validators/video-imports.ts
index ffad482b4..c571f5ddd 100644
--- a/server/helpers/custom-validators/video-imports.ts
+++ b/server/helpers/custom-validators/video-imports.ts
@@ -20,7 +20,7 @@ function isVideoImportTargetUrlValid (url: string) {
20} 20}
21 21
22function isVideoImportStateValid (value: any) { 22function isVideoImportStateValid (value: any) {
23 return exists(value) && VIDEO_IMPORT_STATES[ value ] !== undefined 23 return exists(value) && VIDEO_IMPORT_STATES[value] !== undefined
24} 24}
25 25
26const videoTorrentImportTypes = Object.keys(MIMETYPES.TORRENT.MIMETYPE_EXT).map(m => `(${m})`) 26const videoTorrentImportTypes = Object.keys(MIMETYPES.TORRENT.MIMETYPE_EXT).map(m => `(${m})`)
diff --git a/server/helpers/custom-validators/video-playlists.ts b/server/helpers/custom-validators/video-playlists.ts
index 4bb8384ab..180018fc5 100644
--- a/server/helpers/custom-validators/video-playlists.ts
+++ b/server/helpers/custom-validators/video-playlists.ts
@@ -1,8 +1,6 @@
1import { exists } from './misc' 1import { exists } from './misc'
2import validator from 'validator' 2import validator from 'validator'
3import { CONSTRAINTS_FIELDS, VIDEO_PLAYLIST_PRIVACIES, VIDEO_PLAYLIST_TYPES } from '../../initializers/constants' 3import { CONSTRAINTS_FIELDS, VIDEO_PLAYLIST_PRIVACIES, VIDEO_PLAYLIST_TYPES } from '../../initializers/constants'
4import * as express from 'express'
5import { VideoPlaylistModel } from '../../models/video/video-playlist'
6 4
7const PLAYLISTS_CONSTRAINT_FIELDS = CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS 5const PLAYLISTS_CONSTRAINT_FIELDS = CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS
8 6
@@ -15,7 +13,7 @@ function isVideoPlaylistDescriptionValid (value: any) {
15} 13}
16 14
17function isVideoPlaylistPrivacyValid (value: number) { 15function isVideoPlaylistPrivacyValid (value: number) {
18 return validator.isInt(value + '') && VIDEO_PLAYLIST_PRIVACIES[ value ] !== undefined 16 return validator.isInt(value + '') && VIDEO_PLAYLIST_PRIVACIES[value] !== undefined
19} 17}
20 18
21function isVideoPlaylistTimestampValid (value: any) { 19function isVideoPlaylistTimestampValid (value: any) {
@@ -23,7 +21,7 @@ function isVideoPlaylistTimestampValid (value: any) {
23} 21}
24 22
25function isVideoPlaylistTypeValid (value: any) { 23function isVideoPlaylistTypeValid (value: any) {
26 return exists(value) && VIDEO_PLAYLIST_TYPES[ value ] !== undefined 24 return exists(value) && VIDEO_PLAYLIST_TYPES[value] !== undefined
27} 25}
28 26
29// --------------------------------------------------------------------------- 27// ---------------------------------------------------------------------------
diff --git a/server/helpers/custom-validators/video-redundancies.ts b/server/helpers/custom-validators/video-redundancies.ts
new file mode 100644
index 000000000..50a559c4f
--- /dev/null
+++ b/server/helpers/custom-validators/video-redundancies.ts
@@ -0,0 +1,12 @@
1import { exists } from './misc'
2
3function isVideoRedundancyTarget (value: any) {
4 return exists(value) &&
5 (value === 'my-videos' || value === 'remote-videos')
6}
7
8// ---------------------------------------------------------------------------
9
10export {
11 isVideoRedundancyTarget
12}
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts
index a9e859e54..cfb430c63 100644
--- a/server/helpers/custom-validators/videos.ts
+++ b/server/helpers/custom-validators/videos.ts
@@ -20,15 +20,15 @@ function isVideoFilterValid (filter: VideoFilter) {
20} 20}
21 21
22function isVideoCategoryValid (value: any) { 22function isVideoCategoryValid (value: any) {
23 return value === null || VIDEO_CATEGORIES[ value ] !== undefined 23 return value === null || VIDEO_CATEGORIES[value] !== undefined
24} 24}
25 25
26function isVideoStateValid (value: any) { 26function isVideoStateValid (value: any) {
27 return exists(value) && VIDEO_STATES[ value ] !== undefined 27 return exists(value) && VIDEO_STATES[value] !== undefined
28} 28}
29 29
30function isVideoLicenceValid (value: any) { 30function isVideoLicenceValid (value: any) {
31 return value === null || VIDEO_LICENCES[ value ] !== undefined 31 return value === null || VIDEO_LICENCES[value] !== undefined
32} 32}
33 33
34function isVideoLanguageValid (value: any) { 34function isVideoLanguageValid (value: any) {
@@ -98,7 +98,7 @@ function isVideoImage (files: { [ fieldname: string ]: Express.Multer.File[] } |
98} 98}
99 99
100function isVideoPrivacyValid (value: number) { 100function isVideoPrivacyValid (value: number) {
101 return VIDEO_PRIVACIES[ value ] !== undefined 101 return VIDEO_PRIVACIES[value] !== undefined
102} 102}
103 103
104function isScheduleVideoUpdatePrivacyValid (value: number) { 104function isScheduleVideoUpdatePrivacyValid (value: number) {
diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts
index 9bf6d85a8..f46812977 100644
--- a/server/helpers/express-utils.ts
+++ b/server/helpers/express-utils.ts
@@ -12,7 +12,7 @@ function buildNSFWFilter (res?: express.Response, paramNSFW?: string) {
12 if (paramNSFW === 'false') return false 12 if (paramNSFW === 'false') return false
13 if (paramNSFW === 'both') return undefined 13 if (paramNSFW === 'both') return undefined
14 14
15 if (res && res.locals.oauth) { 15 if (res?.locals.oauth) {
16 const user = res.locals.oauth.token.User 16 const user = res.locals.oauth.token.User
17 17
18 // User does not want NSFW videos 18 // User does not want NSFW videos
@@ -28,7 +28,7 @@ function buildNSFWFilter (res?: express.Response, paramNSFW?: string) {
28 return null 28 return null
29} 29}
30 30
31function cleanUpReqFiles (req: { files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[] }) { 31function cleanUpReqFiles (req: { files: { [fieldname: string]: Express.Multer.File[] } | Express.Multer.File[] }) {
32 const files = req.files 32 const files = req.files
33 33
34 if (!files) return 34 if (!files) return
@@ -39,7 +39,7 @@ function cleanUpReqFiles (req: { files: { [ fieldname: string ]: Express.Multer.
39 } 39 }
40 40
41 for (const key of Object.keys(files)) { 41 for (const key of Object.keys(files)) {
42 const file = files[ key ] 42 const file = files[key]
43 43
44 if (isArray(file)) file.forEach(f => deleteFileAsync(f.path)) 44 if (isArray(file)) file.forEach(f => deleteFileAsync(f.path))
45 else deleteFileAsync(file.path) 45 else deleteFileAsync(file.path)
@@ -65,18 +65,18 @@ function badRequest (req: express.Request, res: express.Response) {
65 65
66function createReqFiles ( 66function createReqFiles (
67 fieldNames: string[], 67 fieldNames: string[],
68 mimeTypes: { [ id: string ]: string }, 68 mimeTypes: { [id: string]: string },
69 destinations: { [ fieldName: string ]: string } 69 destinations: { [fieldName: string]: string }
70) { 70) {
71 const storage = multer.diskStorage({ 71 const storage = multer.diskStorage({
72 destination: (req, file, cb) => { 72 destination: (req, file, cb) => {
73 cb(null, destinations[ file.fieldname ]) 73 cb(null, destinations[file.fieldname])
74 }, 74 },
75 75
76 filename: async (req, file, cb) => { 76 filename: async (req, file, cb) => {
77 let extension: string 77 let extension: string
78 const fileExtension = extname(file.originalname) 78 const fileExtension = extname(file.originalname)
79 const extensionFromMimetype = mimeTypes[ file.mimetype ] 79 const extensionFromMimetype = mimeTypes[file.mimetype]
80 80
81 // Take the file extension if we don't understand the mime type 81 // Take the file extension if we don't understand the mime type
82 // We have the OGG/OGV exception too because firefox sends a bad mime type when sending an OGG file 82 // We have the OGG/OGV exception too because firefox sends a bad mime type when sending an OGG file
@@ -99,7 +99,7 @@ function createReqFiles (
99 } 99 }
100 }) 100 })
101 101
102 let fields: { name: string, maxCount: number }[] = [] 102 const fields: { name: string, maxCount: number }[] = []
103 for (const fieldName of fieldNames) { 103 for (const fieldName of fieldNames) {
104 fields.push({ 104 fields.push({
105 name: fieldName, 105 name: fieldName,
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts
index 00c32e99a..084516e55 100644
--- a/server/helpers/ffmpeg-utils.ts
+++ b/server/helpers/ffmpeg-utils.ts
@@ -1,6 +1,6 @@
1import * as ffmpeg from 'fluent-ffmpeg' 1import * as ffmpeg from 'fluent-ffmpeg'
2import { dirname, join } from 'path' 2import { dirname, join } from 'path'
3import { getTargetBitrate, getMaxBitrate, VideoResolution } from '../../shared/models/videos' 3import { getMaxBitrate, getTargetBitrate, VideoResolution } from '../../shared/models/videos'
4import { FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants' 4import { FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants'
5import { processImage } from './image-utils' 5import { processImage } from './image-utils'
6import { logger } from './logger' 6import { logger } from './logger'
@@ -8,6 +8,71 @@ import { checkFFmpegEncoders } from '../initializers/checker-before-init'
8import { readFile, remove, writeFile } from 'fs-extra' 8import { readFile, remove, writeFile } from 'fs-extra'
9import { CONFIG } from '../initializers/config' 9import { CONFIG } from '../initializers/config'
10 10
11/**
12 * A toolbox to play with audio
13 */
14namespace audio {
15 export const get = (videoPath: string) => {
16 // without position, ffprobe considers the last input only
17 // we make it consider the first input only
18 // if you pass a file path to pos, then ffprobe acts on that file directly
19 return new Promise<{ absolutePath: string, audioStream?: any }>((res, rej) => {
20
21 function parseFfprobe (err: any, data: ffmpeg.FfprobeData) {
22 if (err) return rej(err)
23
24 if ('streams' in data) {
25 const audioStream = data.streams.find(stream => stream['codec_type'] === 'audio')
26 if (audioStream) {
27 return res({
28 absolutePath: data.format.filename,
29 audioStream
30 })
31 }
32 }
33
34 return res({ absolutePath: data.format.filename })
35 }
36
37 return ffmpeg.ffprobe(videoPath, parseFfprobe)
38 })
39 }
40
41 export namespace bitrate {
42 const baseKbitrate = 384
43
44 const toBits = (kbits: number) => kbits * 8000
45
46 export const aac = (bitrate: number): number => {
47 switch (true) {
48 case bitrate > toBits(baseKbitrate):
49 return baseKbitrate
50
51 default:
52 return -1 // we interpret it as a signal to copy the audio stream as is
53 }
54 }
55
56 export const mp3 = (bitrate: number): number => {
57 /*
58 a 192kbit/sec mp3 doesn't hold as much information as a 192kbit/sec aac.
59 That's why, when using aac, we can go to lower kbit/sec. The equivalences
60 made here are not made to be accurate, especially with good mp3 encoders.
61 */
62 switch (true) {
63 case bitrate <= toBits(192):
64 return 128
65
66 case bitrate <= toBits(384):
67 return 256
68
69 default:
70 return baseKbitrate
71 }
72 }
73 }
74}
75
11function computeResolutionsToTranscode (videoFileHeight: number) { 76function computeResolutionsToTranscode (videoFileHeight: number) {
12 const resolutionsEnabled: number[] = [] 77 const resolutionsEnabled: number[] = []
13 const configResolutions = CONFIG.TRANSCODING.RESOLUTIONS 78 const configResolutions = CONFIG.TRANSCODING.RESOLUTIONS
@@ -24,7 +89,7 @@ function computeResolutionsToTranscode (videoFileHeight: number) {
24 ] 89 ]
25 90
26 for (const resolution of resolutions) { 91 for (const resolution of resolutions) {
27 if (configResolutions[ resolution + 'p' ] === true && videoFileHeight > resolution) { 92 if (configResolutions[resolution + 'p'] === true && videoFileHeight > resolution) {
28 resolutionsEnabled.push(resolution) 93 resolutionsEnabled.push(resolution)
29 } 94 }
30 } 95 }
@@ -48,9 +113,9 @@ async function getVideoStreamCodec (path: string) {
48 const videoCodec = videoStream.codec_tag_string 113 const videoCodec = videoStream.codec_tag_string
49 114
50 const baseProfileMatrix = { 115 const baseProfileMatrix = {
51 'High': '6400', 116 High: '6400',
52 'Main': '4D40', 117 Main: '4D40',
53 'Baseline': '42E0' 118 Baseline: '42E0'
54 } 119 }
55 120
56 let baseProfile = baseProfileMatrix[videoStream.profile] 121 let baseProfile = baseProfileMatrix[videoStream.profile]
@@ -91,7 +156,7 @@ async function getVideoFileFPS (path: string) {
91 if (videoStream === null) return 0 156 if (videoStream === null) return 0
92 157
93 for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) { 158 for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) {
94 const valuesText: string = videoStream[ key ] 159 const valuesText: string = videoStream[key]
95 if (!valuesText) continue 160 if (!valuesText) continue
96 161
97 const [ frames, seconds ] = valuesText.split('/') 162 const [ frames, seconds ] = valuesText.split('/')
@@ -191,7 +256,8 @@ interface OnlyAudioTranscodeOptions extends BaseTranscodeOptions {
191 type: 'only-audio' 256 type: 'only-audio'
192} 257}
193 258
194type TranscodeOptions = HLSTranscodeOptions 259type TranscodeOptions =
260 HLSTranscodeOptions
195 | VideoTranscodeOptions 261 | VideoTranscodeOptions
196 | MergeAudioTranscodeOptions 262 | MergeAudioTranscodeOptions
197 | OnlyAudioTranscodeOptions 263 | OnlyAudioTranscodeOptions
@@ -204,13 +270,13 @@ function transcode (options: TranscodeOptions) {
204 .output(options.outputPath) 270 .output(options.outputPath)
205 271
206 if (options.type === 'quick-transcode') { 272 if (options.type === 'quick-transcode') {
207 command = await buildQuickTranscodeCommand(command) 273 command = buildQuickTranscodeCommand(command)
208 } else if (options.type === 'hls') { 274 } else if (options.type === 'hls') {
209 command = await buildHLSCommand(command, options) 275 command = await buildHLSCommand(command, options)
210 } else if (options.type === 'merge-audio') { 276 } else if (options.type === 'merge-audio') {
211 command = await buildAudioMergeCommand(command, options) 277 command = await buildAudioMergeCommand(command, options)
212 } else if (options.type === 'only-audio') { 278 } else if (options.type === 'only-audio') {
213 command = await buildOnlyAudioCommand(command, options) 279 command = buildOnlyAudioCommand(command, options)
214 } else { 280 } else {
215 command = await buildx264Command(command, options) 281 command = await buildx264Command(command, options)
216 } 282 }
@@ -247,22 +313,27 @@ async function canDoQuickTranscode (path: string): Promise<boolean> {
247 313
248 // check video params 314 // check video params
249 if (videoStream == null) return false 315 if (videoStream == null) return false
250 if (videoStream[ 'codec_name' ] !== 'h264') return false 316 if (videoStream['codec_name'] !== 'h264') return false
251 if (videoStream[ 'pix_fmt' ] !== 'yuv420p') return false 317 if (videoStream['pix_fmt'] !== 'yuv420p') return false
252 if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false 318 if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false
253 if (bitRate > getMaxBitrate(resolution.videoFileResolution, fps, VIDEO_TRANSCODING_FPS)) return false 319 if (bitRate > getMaxBitrate(resolution.videoFileResolution, fps, VIDEO_TRANSCODING_FPS)) return false
254 320
255 // check audio params (if audio stream exists) 321 // check audio params (if audio stream exists)
256 if (parsedAudio.audioStream) { 322 if (parsedAudio.audioStream) {
257 if (parsedAudio.audioStream[ 'codec_name' ] !== 'aac') return false 323 if (parsedAudio.audioStream['codec_name'] !== 'aac') return false
258 324
259 const maxAudioBitrate = audio.bitrate[ 'aac' ](parsedAudio.audioStream[ 'bit_rate' ]) 325 const maxAudioBitrate = audio.bitrate['aac'](parsedAudio.audioStream['bit_rate'])
260 if (maxAudioBitrate !== -1 && parsedAudio.audioStream[ 'bit_rate' ] > maxAudioBitrate) return false 326 if (maxAudioBitrate !== -1 && parsedAudio.audioStream['bit_rate'] > maxAudioBitrate) return false
261 } 327 }
262 328
263 return true 329 return true
264} 330}
265 331
332function getClosestFramerateStandard (fps: number, type: 'HD_STANDARD' | 'STANDARD'): number {
333 return VIDEO_TRANSCODING_FPS[type].slice(0)
334 .sort((a, b) => fps % a - fps % b)[0]
335}
336
266// --------------------------------------------------------------------------- 337// ---------------------------------------------------------------------------
267 338
268export { 339export {
@@ -286,13 +357,14 @@ export {
286 357
287async function buildx264Command (command: ffmpeg.FfmpegCommand, options: TranscodeOptions) { 358async function buildx264Command (command: ffmpeg.FfmpegCommand, options: TranscodeOptions) {
288 let fps = await getVideoFileFPS(options.inputPath) 359 let fps = await getVideoFileFPS(options.inputPath)
289 // On small/medium resolutions, limit FPS
290 if ( 360 if (
361 // On small/medium resolutions, limit FPS
291 options.resolution !== undefined && 362 options.resolution !== undefined &&
292 options.resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN && 363 options.resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN &&
293 fps > VIDEO_TRANSCODING_FPS.AVERAGE 364 fps > VIDEO_TRANSCODING_FPS.AVERAGE
294 ) { 365 ) {
295 fps = VIDEO_TRANSCODING_FPS.AVERAGE 366 // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value
367 fps = getClosestFramerateStandard(fps, 'STANDARD')
296 } 368 }
297 369
298 command = await presetH264(command, options.inputPath, options.resolution, fps) 370 command = await presetH264(command, options.inputPath, options.resolution, fps)
@@ -305,7 +377,7 @@ async function buildx264Command (command: ffmpeg.FfmpegCommand, options: Transco
305 377
306 if (fps) { 378 if (fps) {
307 // Hard FPS limits 379 // Hard FPS limits
308 if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = VIDEO_TRANSCODING_FPS.MAX 380 if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard(fps, 'HD_STANDARD')
309 else if (fps < VIDEO_TRANSCODING_FPS.MIN) fps = VIDEO_TRANSCODING_FPS.MIN 381 else if (fps < VIDEO_TRANSCODING_FPS.MIN) fps = VIDEO_TRANSCODING_FPS.MIN
310 382
311 command = command.withFPS(fps) 383 command = command.withFPS(fps)
@@ -327,14 +399,14 @@ async function buildAudioMergeCommand (command: ffmpeg.FfmpegCommand, options: M
327 return command 399 return command
328} 400}
329 401
330async function buildOnlyAudioCommand (command: ffmpeg.FfmpegCommand, options: OnlyAudioTranscodeOptions) { 402function buildOnlyAudioCommand (command: ffmpeg.FfmpegCommand, options: OnlyAudioTranscodeOptions) {
331 command = await presetOnlyAudio(command) 403 command = presetOnlyAudio(command)
332 404
333 return command 405 return command
334} 406}
335 407
336async function buildQuickTranscodeCommand (command: ffmpeg.FfmpegCommand) { 408function buildQuickTranscodeCommand (command: ffmpeg.FfmpegCommand) {
337 command = await presetCopy(command) 409 command = presetCopy(command)
338 410
339 command = command.outputOption('-map_metadata -1') // strip all metadata 411 command = command.outputOption('-map_metadata -1') // strip all metadata
340 .outputOption('-movflags faststart') 412 .outputOption('-movflags faststart')
@@ -345,7 +417,7 @@ async function buildQuickTranscodeCommand (command: ffmpeg.FfmpegCommand) {
345async function buildHLSCommand (command: ffmpeg.FfmpegCommand, options: HLSTranscodeOptions) { 417async function buildHLSCommand (command: ffmpeg.FfmpegCommand, options: HLSTranscodeOptions) {
346 const videoPath = getHLSVideoPath(options) 418 const videoPath = getHLSVideoPath(options)
347 419
348 if (options.copyCodecs) command = await presetCopy(command) 420 if (options.copyCodecs) command = presetCopy(command)
349 else command = await buildx264Command(command, options) 421 else command = await buildx264Command(command, options)
350 422
351 command = command.outputOption('-hls_time 4') 423 command = command.outputOption('-hls_time 4')
@@ -413,71 +485,6 @@ async function presetH264VeryFast (command: ffmpeg.FfmpegCommand, input: string,
413} 485}
414 486
415/** 487/**
416 * A toolbox to play with audio
417 */
418namespace audio {
419 export const get = (videoPath: string) => {
420 // without position, ffprobe considers the last input only
421 // we make it consider the first input only
422 // if you pass a file path to pos, then ffprobe acts on that file directly
423 return new Promise<{ absolutePath: string, audioStream?: any }>((res, rej) => {
424
425 function parseFfprobe (err: any, data: ffmpeg.FfprobeData) {
426 if (err) return rej(err)
427
428 if ('streams' in data) {
429 const audioStream = data.streams.find(stream => stream[ 'codec_type' ] === 'audio')
430 if (audioStream) {
431 return res({
432 absolutePath: data.format.filename,
433 audioStream
434 })
435 }
436 }
437
438 return res({ absolutePath: data.format.filename })
439 }
440
441 return ffmpeg.ffprobe(videoPath, parseFfprobe)
442 })
443 }
444
445 export namespace bitrate {
446 const baseKbitrate = 384
447
448 const toBits = (kbits: number) => kbits * 8000
449
450 export const aac = (bitrate: number): number => {
451 switch (true) {
452 case bitrate > toBits(baseKbitrate):
453 return baseKbitrate
454
455 default:
456 return -1 // we interpret it as a signal to copy the audio stream as is
457 }
458 }
459
460 export const mp3 = (bitrate: number): number => {
461 /*
462 a 192kbit/sec mp3 doesn't hold as much information as a 192kbit/sec aac.
463 That's why, when using aac, we can go to lower kbit/sec. The equivalences
464 made here are not made to be accurate, especially with good mp3 encoders.
465 */
466 switch (true) {
467 case bitrate <= toBits(192):
468 return 128
469
470 case bitrate <= toBits(384):
471 return 256
472
473 default:
474 return baseKbitrate
475 }
476 }
477 }
478}
479
480/**
481 * Standard profile, with variable bitrate audio and faststart. 488 * Standard profile, with variable bitrate audio and faststart.
482 * 489 *
483 * As for the audio, quality '5' is the highest and ensures 96-112kbps/channel 490 * As for the audio, quality '5' is the highest and ensures 96-112kbps/channel
@@ -507,10 +514,10 @@ async function presetH264 (command: ffmpeg.FfmpegCommand, input: string, resolut
507 // of course this is far from perfect, but it might save some space in the end 514 // of course this is far from perfect, but it might save some space in the end
508 localCommand = localCommand.audioCodec('aac') 515 localCommand = localCommand.audioCodec('aac')
509 516
510 const audioCodecName = parsedAudio.audioStream[ 'codec_name' ] 517 const audioCodecName = parsedAudio.audioStream['codec_name']
511 518
512 if (audio.bitrate[ audioCodecName ]) { 519 if (audio.bitrate[audioCodecName]) {
513 const bitrate = audio.bitrate[ audioCodecName ](parsedAudio.audioStream[ 'bit_rate' ]) 520 const bitrate = audio.bitrate[audioCodecName](parsedAudio.audioStream['bit_rate'])
514 if (bitrate !== undefined && bitrate !== -1) localCommand = localCommand.audioBitrate(bitrate) 521 if (bitrate !== undefined && bitrate !== -1) localCommand = localCommand.audioBitrate(bitrate)
515 } 522 }
516 } 523 }
@@ -531,14 +538,14 @@ async function presetH264 (command: ffmpeg.FfmpegCommand, input: string, resolut
531 return localCommand 538 return localCommand
532} 539}
533 540
534async function presetCopy (command: ffmpeg.FfmpegCommand): Promise<ffmpeg.FfmpegCommand> { 541function presetCopy (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
535 return command 542 return command
536 .format('mp4') 543 .format('mp4')
537 .videoCodec('copy') 544 .videoCodec('copy')
538 .audioCodec('copy') 545 .audioCodec('copy')
539} 546}
540 547
541async function presetOnlyAudio (command: ffmpeg.FfmpegCommand): Promise<ffmpeg.FfmpegCommand> { 548function presetOnlyAudio (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
542 return command 549 return command
543 .format('mp4') 550 .format('mp4')
544 .audioCodec('copy') 551 .audioCodec('copy')
diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts
index 395417612..b8ae28b3f 100644
--- a/server/helpers/logger.ts
+++ b/server/helpers/logger.ts
@@ -5,7 +5,7 @@ import * as winston from 'winston'
5import { FileTransportOptions } from 'winston/lib/winston/transports' 5import { FileTransportOptions } from 'winston/lib/winston/transports'
6import { CONFIG } from '../initializers/config' 6import { CONFIG } from '../initializers/config'
7import { omit } from 'lodash' 7import { omit } from 'lodash'
8import { LOG_FILENAME } from '@server/initializers/constants' 8import { LOG_FILENAME } from '../initializers/constants'
9 9
10const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT 10const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
11 11
@@ -27,7 +27,7 @@ function getLoggerReplacer () {
27 if (value instanceof Error) { 27 if (value instanceof Error) {
28 const error = {} 28 const error = {}
29 29
30 Object.getOwnPropertyNames(value).forEach(key => error[ key ] = value[ key ]) 30 Object.getOwnPropertyNames(value).forEach(key => { error[key] = value[key] })
31 31
32 return error 32 return error
33 } 33 }
@@ -98,19 +98,20 @@ function bunyanLogFactory (level: string) {
98 let args: any[] = [] 98 let args: any[] = []
99 args.concat(arguments) 99 args.concat(arguments)
100 100
101 if (arguments[ 0 ] instanceof Error) { 101 if (arguments[0] instanceof Error) {
102 meta = arguments[ 0 ].toString() 102 meta = arguments[0].toString()
103 args = Array.prototype.slice.call(arguments, 1) 103 args = Array.prototype.slice.call(arguments, 1)
104 args.push(meta) 104 args.push(meta)
105 } else if (typeof (args[ 0 ]) !== 'string') { 105 } else if (typeof (args[0]) !== 'string') {
106 meta = arguments[ 0 ] 106 meta = arguments[0]
107 args = Array.prototype.slice.call(arguments, 1) 107 args = Array.prototype.slice.call(arguments, 1)
108 args.push(meta) 108 args.push(meta)
109 } 109 }
110 110
111 logger[ level ].apply(logger, args) 111 logger[level].apply(logger, args)
112 } 112 }
113} 113}
114
114const bunyanLogger = { 115const bunyanLogger = {
115 trace: bunyanLogFactory('debug'), 116 trace: bunyanLogFactory('debug'),
116 debug: bunyanLogFactory('debug'), 117 debug: bunyanLogFactory('debug'),
diff --git a/server/helpers/middlewares/videos.ts b/server/helpers/middlewares/videos.ts
index 74f529804..409f78650 100644
--- a/server/helpers/middlewares/videos.ts
+++ b/server/helpers/middlewares/videos.ts
@@ -2,7 +2,16 @@ import { Response } from 'express'
2import { fetchVideo, VideoFetchType } from '../video' 2import { fetchVideo, VideoFetchType } from '../video'
3import { UserRight } from '../../../shared/models/users' 3import { UserRight } from '../../../shared/models/users'
4import { VideoChannelModel } from '../../models/video/video-channel' 4import { VideoChannelModel } from '../../models/video/video-channel'
5import { MUser, MUserAccountId, MVideoAccountLight, MVideoFullLight, MVideoThumbnail, MVideoWithRights } from '@server/typings/models' 5import {
6 MUser,
7 MUserAccountId,
8 MVideoAccountLight,
9 MVideoFullLight,
10 MVideoIdThumbnail,
11 MVideoImmutable,
12 MVideoThumbnail,
13 MVideoWithRights
14} from '@server/typings/models'
6 15
7async function doesVideoExist (id: number | string, res: Response, fetchType: VideoFetchType = 'all') { 16async function doesVideoExist (id: number | string, res: Response, fetchType: VideoFetchType = 'all') {
8 const userId = res.locals.oauth ? res.locals.oauth.token.User.id : undefined 17 const userId = res.locals.oauth ? res.locals.oauth.token.User.id : undefined
@@ -22,8 +31,12 @@ async function doesVideoExist (id: number | string, res: Response, fetchType: Vi
22 res.locals.videoAll = video as MVideoFullLight 31 res.locals.videoAll = video as MVideoFullLight
23 break 32 break
24 33
34 case 'only-immutable-attributes':
35 res.locals.onlyImmutableVideo = video as MVideoImmutable
36 break
37
25 case 'id': 38 case 'id':
26 res.locals.videoId = video 39 res.locals.videoId = video as MVideoIdThumbnail
27 break 40 break
28 41
29 case 'only-video': 42 case 'only-video':
diff --git a/server/helpers/regexp.ts b/server/helpers/regexp.ts
index 2336654b0..cfc2be488 100644
--- a/server/helpers/regexp.ts
+++ b/server/helpers/regexp.ts
@@ -1,8 +1,8 @@
1// Thanks to https://regex101.com 1// Thanks to https://regex101.com
2function regexpCapture (str: string, regex: RegExp, maxIterations = 100) { 2function regexpCapture (str: string, regex: RegExp, maxIterations = 100) {
3 const result: RegExpExecArray[] = []
3 let m: RegExpExecArray 4 let m: RegExpExecArray
4 let i = 0 5 let i = 0
5 let result: RegExpExecArray[] = []
6 6
7 // tslint:disable:no-conditional-assignment 7 // tslint:disable:no-conditional-assignment
8 while ((m = regex.exec(str)) !== null && i < maxIterations) { 8 while ((m = regex.exec(str)) !== null && i < maxIterations) {
diff --git a/server/helpers/register-ts-paths.ts b/server/helpers/register-ts-paths.ts
index e8db369e3..eec7fed3e 100644
--- a/server/helpers/register-ts-paths.ts
+++ b/server/helpers/register-ts-paths.ts
@@ -1,5 +1,5 @@
1import { resolve } from 'path' 1import { resolve } from 'path'
2const tsConfigPaths = require('tsconfig-paths') 2import tsConfigPaths = require('tsconfig-paths')
3 3
4const tsConfig = require('../../tsconfig.json') 4const tsConfig = require('../../tsconfig.json')
5 5
diff --git a/server/helpers/signup.ts b/server/helpers/signup.ts
index 7c73f7c5c..d34ff2db5 100644
--- a/server/helpers/signup.ts
+++ b/server/helpers/signup.ts
@@ -21,7 +21,7 @@ async function isSignupAllowed (): Promise<{ allowed: boolean, errorMessage?: st
21 21
22function isSignupAllowedForCurrentIP (ip: string) { 22function isSignupAllowedForCurrentIP (ip: string) {
23 const addr = ipaddr.parse(ip) 23 const addr = ipaddr.parse(ip)
24 let excludeList = [ 'blacklist' ] 24 const excludeList = [ 'blacklist' ]
25 let matched = '' 25 let matched = ''
26 26
27 // if there is a valid, non-empty whitelist, we exclude all unknown adresses too 27 // if there is a valid, non-empty whitelist, we exclude all unknown adresses too
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts
index 4c6f200f8..7a4c781cc 100644
--- a/server/helpers/utils.ts
+++ b/server/helpers/utils.ts
@@ -1,6 +1,6 @@
1import { ResultList } from '../../shared' 1import { ResultList } from '../../shared'
2import { ApplicationModel } from '../models/application/application' 2import { ApplicationModel } from '../models/application/application'
3import { execPromise, execPromise2, pseudoRandomBytesPromise, sha256 } from './core-utils' 3import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils'
4import { logger } from './logger' 4import { logger } from './logger'
5import { join } from 'path' 5import { join } from 'path'
6import { Instance as ParseTorrent } from 'parse-torrent' 6import { Instance as ParseTorrent } from 'parse-torrent'
@@ -14,7 +14,7 @@ function deleteFileAsync (path: string) {
14} 14}
15 15
16async function generateRandomString (size: number) { 16async function generateRandomString (size: number) {
17 const raw = await pseudoRandomBytesPromise(size) 17 const raw = await randomBytesPromise(size)
18 18
19 return raw.toString('hex') 19 return raw.toString('hex')
20} 20}
diff --git a/server/helpers/video.ts b/server/helpers/video.ts
index 5b9c026b1..4fe2a60f0 100644
--- a/server/helpers/video.ts
+++ b/server/helpers/video.ts
@@ -5,13 +5,15 @@ import {
5 MVideoFullLight, 5 MVideoFullLight,
6 MVideoIdThumbnail, 6 MVideoIdThumbnail,
7 MVideoThumbnail, 7 MVideoThumbnail,
8 MVideoWithRights 8 MVideoWithRights,
9 MVideoImmutable
9} from '@server/typings/models' 10} from '@server/typings/models'
10import { Response } from 'express' 11import { Response } from 'express'
11 12
12type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none' 13type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none' | 'only-immutable-attributes'
13 14
14function fetchVideo (id: number | string, fetchType: 'all', userId?: number): Bluebird<MVideoFullLight> 15function fetchVideo (id: number | string, fetchType: 'all', userId?: number): Bluebird<MVideoFullLight>
16function fetchVideo (id: number | string, fetchType: 'only-immutable-attributes'): Bluebird<MVideoImmutable>
15function fetchVideo (id: number | string, fetchType: 'only-video', userId?: number): Bluebird<MVideoThumbnail> 17function fetchVideo (id: number | string, fetchType: 'only-video', userId?: number): Bluebird<MVideoThumbnail>
16function fetchVideo (id: number | string, fetchType: 'only-video-with-rights', userId?: number): Bluebird<MVideoWithRights> 18function fetchVideo (id: number | string, fetchType: 'only-video-with-rights', userId?: number): Bluebird<MVideoWithRights>
17function fetchVideo (id: number | string, fetchType: 'id' | 'none', userId?: number): Bluebird<MVideoIdThumbnail> 19function fetchVideo (id: number | string, fetchType: 'id' | 'none', userId?: number): Bluebird<MVideoIdThumbnail>
@@ -19,14 +21,16 @@ function fetchVideo (
19 id: number | string, 21 id: number | string,
20 fetchType: VideoFetchType, 22 fetchType: VideoFetchType,
21 userId?: number 23 userId?: number
22): Bluebird<MVideoFullLight | MVideoThumbnail | MVideoWithRights | MVideoIdThumbnail> 24): Bluebird<MVideoFullLight | MVideoThumbnail | MVideoWithRights | MVideoIdThumbnail | MVideoImmutable>
23function fetchVideo ( 25function fetchVideo (
24 id: number | string, 26 id: number | string,
25 fetchType: VideoFetchType, 27 fetchType: VideoFetchType,
26 userId?: number 28 userId?: number
27): Bluebird<MVideoFullLight | MVideoThumbnail | MVideoWithRights | MVideoIdThumbnail> { 29): Bluebird<MVideoFullLight | MVideoThumbnail | MVideoWithRights | MVideoIdThumbnail | MVideoImmutable> {
28 if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId) 30 if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId)
29 31
32 if (fetchType === 'only-immutable-attributes') return VideoModel.loadImmutableAttributes(id)
33
30 if (fetchType === 'only-video-with-rights') return VideoModel.loadWithRights(id) 34 if (fetchType === 'only-video-with-rights') return VideoModel.loadWithRights(id)
31 35
32 if (fetchType === 'only-video') return VideoModel.load(id) 36 if (fetchType === 'only-video') return VideoModel.load(id)
@@ -34,14 +38,23 @@ function fetchVideo (
34 if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id) 38 if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id)
35} 39}
36 40
37type VideoFetchByUrlType = 'all' | 'only-video' 41type VideoFetchByUrlType = 'all' | 'only-video' | 'only-immutable-attributes'
38 42
39function fetchVideoByUrl (url: string, fetchType: 'all'): Bluebird<MVideoAccountLightBlacklistAllFiles> 43function fetchVideoByUrl (url: string, fetchType: 'all'): Bluebird<MVideoAccountLightBlacklistAllFiles>
44function fetchVideoByUrl (url: string, fetchType: 'only-immutable-attributes'): Bluebird<MVideoImmutable>
40function fetchVideoByUrl (url: string, fetchType: 'only-video'): Bluebird<MVideoThumbnail> 45function fetchVideoByUrl (url: string, fetchType: 'only-video'): Bluebird<MVideoThumbnail>
41function fetchVideoByUrl (url: string, fetchType: VideoFetchByUrlType): Bluebird<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail> 46function fetchVideoByUrl (
42function fetchVideoByUrl (url: string, fetchType: VideoFetchByUrlType): Bluebird<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail> { 47 url: string,
48 fetchType: VideoFetchByUrlType
49): Bluebird<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable>
50function fetchVideoByUrl (
51 url: string,
52 fetchType: VideoFetchByUrlType
53): Bluebird<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
43 if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url) 54 if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url)
44 55
56 if (fetchType === 'only-immutable-attributes') return VideoModel.loadByUrlImmutableAttributes(url)
57
45 if (fetchType === 'only-video') return VideoModel.loadByUrl(url) 58 if (fetchType === 'only-video') return VideoModel.loadByUrl(url)
46} 59}
47 60
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index 3a99518c6..b25e44fcd 100644
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -9,12 +9,12 @@ import { promisify2 } from './core-utils'
9import { MVideo } from '@server/typings/models/video/video' 9import { MVideo } from '@server/typings/models/video/video'
10import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/typings/models/video/video-file' 10import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/typings/models/video/video-file'
11import { isStreamingPlaylist, MStreamingPlaylistVideo } from '@server/typings/models/video/video-streaming-playlist' 11import { isStreamingPlaylist, MStreamingPlaylistVideo } from '@server/typings/models/video/video-streaming-playlist'
12import { STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' 12import { WEBSERVER } from '@server/initializers/constants'
13import * as parseTorrent from 'parse-torrent' 13import * as parseTorrent from 'parse-torrent'
14import * as magnetUtil from 'magnet-uri' 14import * as magnetUtil from 'magnet-uri'
15import { isArray } from '@server/helpers/custom-validators/misc' 15import { isArray } from '@server/helpers/custom-validators/misc'
16import { extractVideo } from '@server/lib/videos' 16import { extractVideo } from '@server/lib/videos'
17import { getTorrentFileName, getVideoFilename, getVideoFilePath } from '@server/lib/video-paths' 17import { getTorrentFileName, getVideoFilePath } from '@server/lib/video-paths'
18 18
19const createTorrentPromise = promisify2<string, any, any>(createTorrent) 19const createTorrentPromise = promisify2<string, any, any>(createTorrent)
20 20
@@ -39,7 +39,7 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName
39 if (torrent.files.length !== 1) { 39 if (torrent.files.length !== 1) {
40 if (timer) clearTimeout(timer) 40 if (timer) clearTimeout(timer)
41 41
42 for (let file of torrent.files) { 42 for (const file of torrent.files) {
43 deleteDownloadedFile({ directoryPath, filepath: file.path }) 43 deleteDownloadedFile({ directoryPath, filepath: file.path })
44 } 44 }
45 45
@@ -47,15 +47,16 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName
47 .then(() => rej(new Error('Cannot import torrent ' + torrentId + ': there are multiple files in it'))) 47 .then(() => rej(new Error('Cannot import torrent ' + torrentId + ': there are multiple files in it')))
48 } 48 }
49 49
50 file = torrent.files[ 0 ] 50 file = torrent.files[0]
51 51
52 // FIXME: avoid creating another stream when https://github.com/webtorrent/webtorrent/issues/1517 is fixed 52 // FIXME: avoid creating another stream when https://github.com/webtorrent/webtorrent/issues/1517 is fixed
53 const writeStream = createWriteStream(path) 53 const writeStream = createWriteStream(path)
54 writeStream.on('finish', () => { 54 writeStream.on('finish', () => {
55 if (timer) clearTimeout(timer) 55 if (timer) clearTimeout(timer)
56 56
57 return safeWebtorrentDestroy(webtorrent, torrentId, { directoryPath, filepath: file.path }, target.torrentName) 57 safeWebtorrentDestroy(webtorrent, torrentId, { directoryPath, filepath: file.path }, target.torrentName)
58 .then(() => res(path)) 58 .then(() => res(path))
59 .catch(err => logger.error('Cannot destroy webtorrent.', { err }))
59 }) 60 })
60 61
61 file.createReadStream().pipe(writeStream) 62 file.createReadStream().pipe(writeStream)
@@ -63,9 +64,16 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName
63 64
64 torrent.on('error', err => rej(err)) 65 torrent.on('error', err => rej(err))
65 66
66 timer = setTimeout(async () => { 67 timer = setTimeout(() => {
67 return safeWebtorrentDestroy(webtorrent, torrentId, file ? { directoryPath, filepath: file.path } : undefined, target.torrentName) 68 const err = new Error('Webtorrent download timeout.')
68 .then(() => rej(new Error('Webtorrent download timeout.'))) 69
70 safeWebtorrentDestroy(webtorrent, torrentId, file ? { directoryPath, filepath: file.path } : undefined, target.torrentName)
71 .then(() => rej(err))
72 .catch(destroyErr => {
73 logger.error('Cannot destroy webtorrent.', { err: destroyErr })
74 rej(err)
75 })
76
69 }, timeout) 77 }, timeout)
70 }) 78 })
71} 79}
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
index 577a59dbf..fc9d416a1 100644
--- a/server/helpers/youtube-dl.ts
+++ b/server/helpers/youtube-dl.ts
@@ -24,20 +24,23 @@ const processOptions = {
24} 24}
25 25
26function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo> { 26function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo> {
27 return new Promise<YoutubeDLInfo>(async (res, rej) => { 27 return new Promise<YoutubeDLInfo>((res, rej) => {
28 let args = opts || [ '-j', '--flat-playlist' ] 28 let args = opts || [ '-j', '--flat-playlist' ]
29 args = wrapWithProxyOptions(args) 29 args = wrapWithProxyOptions(args)
30 30
31 const youtubeDL = await safeGetYoutubeDL() 31 safeGetYoutubeDL()
32 youtubeDL.getInfo(url, args, processOptions, (err, info) => { 32 .then(youtubeDL => {
33 if (err) return rej(err) 33 youtubeDL.getInfo(url, args, processOptions, (err, info) => {
34 if (info.is_live === true) return rej(new Error('Cannot download a live streaming.')) 34 if (err) return rej(err)
35 if (info.is_live === true) return rej(new Error('Cannot download a live streaming.'))
35 36
36 const obj = buildVideoInfo(normalizeObject(info)) 37 const obj = buildVideoInfo(normalizeObject(info))
37 if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video' 38 if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video'
38 39
39 return res(obj) 40 return res(obj)
40 }) 41 })
42 })
43 .catch(err => rej(err))
41 }) 44 })
42} 45}
43 46
@@ -54,26 +57,34 @@ function downloadYoutubeDLVideo (url: string, timeout: number) {
54 options = options.concat([ '--ffmpeg-location', process.env.FFMPEG_PATH ]) 57 options = options.concat([ '--ffmpeg-location', process.env.FFMPEG_PATH ])
55 } 58 }
56 59
57 return new Promise<string>(async (res, rej) => { 60 return new Promise<string>((res, rej) => {
58 const youtubeDL = await safeGetYoutubeDL() 61 safeGetYoutubeDL()
59 youtubeDL.exec(url, options, processOptions, err => { 62 .then(youtubeDL => {
60 clearTimeout(timer) 63 youtubeDL.exec(url, options, processOptions, err => {
64 clearTimeout(timer)
61 65
62 if (err) { 66 if (err) {
63 remove(path) 67 remove(path)
64 .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err })) 68 .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err }))
65 69
66 return rej(err) 70 return rej(err)
67 } 71 }
68 72
69 return res(path) 73 return res(path)
70 }) 74 })
71 75
72 timer = setTimeout(async () => { 76 timer = setTimeout(() => {
73 await remove(path) 77 const err = new Error('YoutubeDL download timeout.')
74 78
75 return rej(new Error('YoutubeDL download timeout.')) 79 remove(path)
76 }, timeout) 80 .finally(() => rej(err))
81 .catch(err => {
82 logger.error('Cannot remove %s in youtubeDL timeout.', path, { err })
83 return rej(err)
84 })
85 }, timeout)
86 })
87 .catch(err => rej(err))
77 }) 88 })
78} 89}
79 90
@@ -103,7 +114,7 @@ async function updateYoutubeDLBinary () {
103 114
104 const url = result.headers.location 115 const url = result.headers.location
105 const downloadFile = request.get(url) 116 const downloadFile = request.get(url)
106 const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[ 1 ] 117 const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[1]
107 118
108 downloadFile.on('response', result => { 119 downloadFile.on('response', result => {
109 if (result.statusCode !== 200) { 120 if (result.statusCode !== 200) {