aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/helpers
diff options
context:
space:
mode:
Diffstat (limited to 'server/helpers')
-rw-r--r--server/helpers/activitypub.ts80
-rw-r--r--server/helpers/audit-logger.ts26
-rw-r--r--server/helpers/core-utils.ts68
-rw-r--r--server/helpers/custom-jsonld-signature.ts90
-rw-r--r--server/helpers/custom-validators/activitypub/actor.ts8
-rw-r--r--server/helpers/custom-validators/activitypub/cache-file.ts2
-rw-r--r--server/helpers/custom-validators/activitypub/video-comments.ts2
-rw-r--r--server/helpers/custom-validators/activitypub/videos.ts47
-rw-r--r--server/helpers/custom-validators/feeds.ts2
-rw-r--r--server/helpers/custom-validators/logs.ts2
-rw-r--r--server/helpers/custom-validators/misc.ts4
-rw-r--r--server/helpers/custom-validators/plugins.ts6
-rw-r--r--server/helpers/custom-validators/user-notifications.ts3
-rw-r--r--server/helpers/custom-validators/users.ts11
-rw-r--r--server/helpers/custom-validators/video-abuses.ts14
-rw-r--r--server/helpers/custom-validators/video-captions.ts8
-rw-r--r--server/helpers/custom-validators/video-imports.ts8
-rw-r--r--server/helpers/custom-validators/video-playlists.ts6
-rw-r--r--server/helpers/custom-validators/video-redundancies.ts12
-rw-r--r--server/helpers/custom-validators/videos.ts10
-rw-r--r--server/helpers/express-utils.ts16
-rw-r--r--server/helpers/ffmpeg-utils.ts228
-rw-r--r--server/helpers/logger.ts66
-rw-r--r--server/helpers/middlewares/video-abuses.ts12
-rw-r--r--server/helpers/middlewares/videos.ts31
-rw-r--r--server/helpers/peertube-crypto.ts8
-rw-r--r--server/helpers/regexp.ts2
-rw-r--r--server/helpers/register-ts-paths.ts2
-rw-r--r--server/helpers/signup.ts2
-rw-r--r--server/helpers/utils.ts29
-rw-r--r--server/helpers/video.ts63
-rw-r--r--server/helpers/webtorrent.ts26
-rw-r--r--server/helpers/youtube-dl.ts129
33 files changed, 630 insertions, 393 deletions
diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts
index 239d8291d..aeb8fde01 100644
--- a/server/helpers/activitypub.ts
+++ b/server/helpers/activitypub.ts
@@ -2,21 +2,35 @@ import * as Bluebird from 'bluebird'
2import validator from 'validator' 2import validator from 'validator'
3import { ResultList } from '../../shared/models' 3import { ResultList } from '../../shared/models'
4import { Activity } from '../../shared/models/activitypub' 4import { Activity } from '../../shared/models/activitypub'
5import { ACTIVITY_PUB } from '../initializers/constants' 5import { ACTIVITY_PUB, REMOTE_SCHEME } from '../initializers/constants'
6import { signJsonLDObject } from './peertube-crypto' 6import { signJsonLDObject } from './peertube-crypto'
7import { pageToStartAndCount } from './core-utils' 7import { pageToStartAndCount } from './core-utils'
8import { parse } from 'url' 8import { URL } from 'url'
9import { MActor } from '../typings/models' 9import { MActor, MVideoAccountLight } from '../typings/models'
10 10import { ContextType } from '@shared/models/activitypub/context'
11function activityPubContextify <T> (data: T) { 11
12 return Object.assign(data, { 12function getContextData (type: ContextType) {
13 '@context': [ 13 const context: any[] = [
14 'https://www.w3.org/ns/activitystreams', 14 'https://www.w3.org/ns/activitystreams',
15 'https://w3id.org/security/v1', 15 'https://w3id.org/security/v1',
16 { 16 {
17 RsaSignature2017: 'https://w3id.org/security#RsaSignature2017', 17 RsaSignature2017: 'https://w3id.org/security#RsaSignature2017'
18 pt: 'https://joinpeertube.org/ns#', 18 }
19 sc: 'http://schema.org#', 19 ]
20
21 if (type !== 'View' && type !== 'Announce') {
22 const additional = {
23 pt: 'https://joinpeertube.org/ns#',
24 sc: 'http://schema.org#'
25 }
26
27 if (type === 'CacheFile') {
28 Object.assign(additional, {
29 expires: 'sc:expires',
30 CacheFile: 'pt:CacheFile'
31 })
32 } else {
33 Object.assign(additional, {
20 Hashtag: 'as:Hashtag', 34 Hashtag: 'as:Hashtag',
21 uuid: 'sc:identifier', 35 uuid: 'sc:identifier',
22 category: 'sc:category', 36 category: 'sc:category',
@@ -24,8 +38,7 @@ function activityPubContextify <T> (data: T) {
24 subtitleLanguage: 'sc:subtitleLanguage', 38 subtitleLanguage: 'sc:subtitleLanguage',
25 sensitive: 'as:sensitive', 39 sensitive: 'as:sensitive',
26 language: 'sc:inLanguage', 40 language: 'sc:inLanguage',
27 expires: 'sc:expires', 41
28 CacheFile: 'pt:CacheFile',
29 Infohash: 'pt:Infohash', 42 Infohash: 'pt:Infohash',
30 originallyPublishedAt: 'sc:datePublished', 43 originallyPublishedAt: 'sc:datePublished',
31 views: { 44 views: {
@@ -71,9 +84,7 @@ function activityPubContextify <T> (data: T) {
71 support: { 84 support: {
72 '@type': 'sc:Text', 85 '@type': 'sc:Text',
73 '@id': 'pt:support' 86 '@id': 'pt:support'
74 } 87 },
75 },
76 {
77 likes: { 88 likes: {
78 '@id': 'as:likes', 89 '@id': 'as:likes',
79 '@type': '@id' 90 '@type': '@id'
@@ -94,9 +105,19 @@ function activityPubContextify <T> (data: T) {
94 '@id': 'as:comments', 105 '@id': 'as:comments',
95 '@type': '@id' 106 '@type': '@id'
96 } 107 }
97 } 108 })
98 ] 109 }
99 }) 110
111 context.push(additional)
112 }
113
114 return {
115 '@context': context
116 }
117}
118
119function activityPubContextify <T> (data: T, type: ContextType = 'All') {
120 return Object.assign({}, data, getContextData(type))
100} 121}
101 122
102type ActivityPubCollectionPaginationHandler = (start: number, count: number) => Bluebird<ResultList<any>> | Promise<ResultList<any>> 123type ActivityPubCollectionPaginationHandler = (start: number, count: number) => Bluebird<ResultList<any>> | Promise<ResultList<any>>
@@ -148,8 +169,8 @@ async function activityPubCollectionPagination (
148 169
149} 170}
150 171
151function buildSignedActivity (byActor: MActor, data: Object) { 172function buildSignedActivity (byActor: MActor, data: Object, contextType?: ContextType) {
152 const activity = activityPubContextify(data) 173 const activity = activityPubContextify(data, contextType)
153 174
154 return signJsonLDObject(byActor, activity) as Promise<Activity> 175 return signJsonLDObject(byActor, activity) as Promise<Activity>
155} 176}
@@ -161,12 +182,18 @@ function getAPId (activity: string | { id: string }) {
161} 182}
162 183
163function checkUrlsSameHost (url1: string, url2: string) { 184function checkUrlsSameHost (url1: string, url2: string) {
164 const idHost = parse(url1).host 185 const idHost = new URL(url1).host
165 const actorHost = parse(url2).host 186 const actorHost = new URL(url2).host
166 187
167 return idHost && actorHost && idHost.toLowerCase() === actorHost.toLowerCase() 188 return idHost && actorHost && idHost.toLowerCase() === actorHost.toLowerCase()
168} 189}
169 190
191function buildRemoteVideoBaseUrl (video: MVideoAccountLight, path: string) {
192 const host = video.VideoChannel.Account.Actor.Server.host
193
194 return REMOTE_SCHEME.HTTP + '://' + host + path
195}
196
170// --------------------------------------------------------------------------- 197// ---------------------------------------------------------------------------
171 198
172export { 199export {
@@ -174,5 +201,6 @@ export {
174 getAPId, 201 getAPId,
175 activityPubContextify, 202 activityPubContextify,
176 activityPubCollectionPagination, 203 activityPubCollectionPagination,
177 buildSignedActivity 204 buildSignedActivity,
205 buildRemoteVideoBaseUrl
178} 206}
diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts
index 9b258dc3a..0bbfbc753 100644
--- a/server/helpers/audit-logger.ts
+++ b/server/helpers/audit-logger.ts
@@ -36,7 +36,7 @@ const auditLogger = winston.createLogger({
36 maxFiles: 5, 36 maxFiles: 5,
37 format: winston.format.combine( 37 format: winston.format.combine(
38 winston.format.timestamp(), 38 winston.format.timestamp(),
39 labelFormatter, 39 labelFormatter(),
40 winston.format.splat(), 40 winston.format.splat(),
41 jsonLoggerFormat 41 jsonLoggerFormat
42 ) 42 )
@@ -81,7 +81,8 @@ function auditLoggerFactory (domain: string) {
81} 81}
82 82
83abstract class EntityAuditView { 83abstract class EntityAuditView {
84 constructor (private keysToKeep: Array<string>, private prefix: string, private entityInfos: object) { } 84 constructor (private readonly keysToKeep: string[], private readonly prefix: string, private readonly entityInfos: object) { }
85
85 toLogKeys (): object { 86 toLogKeys (): object {
86 return chain(flatten(this.entityInfos, { delimiter: '-', safe: true })) 87 return chain(flatten(this.entityInfos, { delimiter: '-', safe: true }))
87 .pick(this.keysToKeep) 88 .pick(this.keysToKeep)
@@ -121,7 +122,7 @@ const videoKeysToKeep = [
121 'downloadEnabled' 122 'downloadEnabled'
122] 123]
123class VideoAuditView extends EntityAuditView { 124class VideoAuditView extends EntityAuditView {
124 constructor (private video: VideoDetails) { 125 constructor (private readonly video: VideoDetails) {
125 super(videoKeysToKeep, 'video', video) 126 super(videoKeysToKeep, 'video', video)
126 } 127 }
127} 128}
@@ -132,7 +133,7 @@ const videoImportKeysToKeep = [
132 'video-name' 133 'video-name'
133] 134]
134class VideoImportAuditView extends EntityAuditView { 135class VideoImportAuditView extends EntityAuditView {
135 constructor (private videoImport: VideoImport) { 136 constructor (private readonly videoImport: VideoImport) {
136 super(videoImportKeysToKeep, 'video-import', videoImport) 137 super(videoImportKeysToKeep, 'video-import', videoImport)
137 } 138 }
138} 139}
@@ -151,7 +152,7 @@ const commentKeysToKeep = [
151 'account-name' 152 'account-name'
152] 153]
153class CommentAuditView extends EntityAuditView { 154class CommentAuditView extends EntityAuditView {
154 constructor (private comment: VideoComment) { 155 constructor (private readonly comment: VideoComment) {
155 super(commentKeysToKeep, 'comment', comment) 156 super(commentKeysToKeep, 'comment', comment)
156 } 157 }
157} 158}
@@ -180,7 +181,7 @@ const userKeysToKeep = [
180 'videoChannels' 181 'videoChannels'
181] 182]
182class UserAuditView extends EntityAuditView { 183class UserAuditView extends EntityAuditView {
183 constructor (private user: User) { 184 constructor (private readonly user: User) {
184 super(userKeysToKeep, 'user', user) 185 super(userKeysToKeep, 'user', user)
185 } 186 }
186} 187}
@@ -206,7 +207,7 @@ const channelKeysToKeep = [
206 'ownerAccount-displayedName' 207 'ownerAccount-displayedName'
207] 208]
208class VideoChannelAuditView extends EntityAuditView { 209class VideoChannelAuditView extends EntityAuditView {
209 constructor (private channel: VideoChannel) { 210 constructor (private readonly channel: VideoChannel) {
210 super(channelKeysToKeep, 'channel', channel) 211 super(channelKeysToKeep, 'channel', channel)
211 } 212 }
212} 213}
@@ -221,7 +222,7 @@ const videoAbuseKeysToKeep = [
221 'createdAt' 222 'createdAt'
222] 223]
223class VideoAbuseAuditView extends EntityAuditView { 224class VideoAbuseAuditView extends EntityAuditView {
224 constructor (private videoAbuse: VideoAbuse) { 225 constructor (private readonly videoAbuse: VideoAbuse) {
225 super(videoAbuseKeysToKeep, 'abuse', videoAbuse) 226 super(videoAbuseKeysToKeep, 'abuse', videoAbuse)
226 } 227 }
227} 228}
@@ -253,9 +254,12 @@ class CustomConfigAuditView extends EntityAuditView {
253 const infos: any = customConfig 254 const infos: any = customConfig
254 const resolutionsDict = infos.transcoding.resolutions 255 const resolutionsDict = infos.transcoding.resolutions
255 const resolutionsArray = [] 256 const resolutionsArray = []
256 Object.entries(resolutionsDict).forEach(([resolution, isEnabled]) => { 257
257 if (isEnabled) resolutionsArray.push(resolution) 258 Object.entries(resolutionsDict)
258 }) 259 .forEach(([ resolution, isEnabled ]) => {
260 if (isEnabled) resolutionsArray.push(resolution)
261 })
262
259 Object.assign({}, infos, { transcoding: { resolutions: resolutionsArray } }) 263 Object.assign({}, infos, { transcoding: { resolutions: resolutionsArray } })
260 super(customConfigKeysToKeep, 'config', infos) 264 super(customConfigKeysToKeep, 'config', infos)
261 } 265 }
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index 7e8252aa4..b1f5d9610 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -1,9 +1,11 @@
1/* eslint-disable no-useless-call */
2
1/* 3/*
2 Different from 'utils' because we don't not import other PeerTube modules. 4 Different from 'utils' because we don't import other PeerTube modules.
3 Useful to avoid circular dependencies. 5 Useful to avoid circular dependencies.
4*/ 6*/
5 7
6import { createHash, HexBase64Latin1Encoding, pseudoRandomBytes } from 'crypto' 8import { createHash, HexBase64Latin1Encoding, randomBytes } from 'crypto'
7import { basename, isAbsolute, join, resolve } from 'path' 9import { basename, isAbsolute, join, resolve } from 'path'
8import * as pem from 'pem' 10import * as pem from 'pem'
9import { URL } from 'url' 11import { URL } from 'url'
@@ -22,31 +24,31 @@ const objectConverter = (oldObject: any, keyConverter: (e: string) => string, va
22 const newObject = {} 24 const newObject = {}
23 Object.keys(oldObject).forEach(oldKey => { 25 Object.keys(oldObject).forEach(oldKey => {
24 const newKey = keyConverter(oldKey) 26 const newKey = keyConverter(oldKey)
25 newObject[ newKey ] = objectConverter(oldObject[ oldKey ], keyConverter, valueConverter) 27 newObject[newKey] = objectConverter(oldObject[oldKey], keyConverter, valueConverter)
26 }) 28 })
27 29
28 return newObject 30 return newObject
29} 31}
30 32
31const timeTable = { 33const timeTable = {
32 ms: 1, 34 ms: 1,
33 second: 1000, 35 second: 1000,
34 minute: 60000, 36 minute: 60000,
35 hour: 3600000, 37 hour: 3600000,
36 day: 3600000 * 24, 38 day: 3600000 * 24,
37 week: 3600000 * 24 * 7, 39 week: 3600000 * 24 * 7,
38 month: 3600000 * 24 * 30 40 month: 3600000 * 24 * 30
39} 41}
40 42
41export function parseDurationToMs (duration: number | string): number { 43export function parseDurationToMs (duration: number | string): number {
42 if (typeof duration === 'number') return duration 44 if (typeof duration === 'number') return duration
43 45
44 if (typeof duration === 'string') { 46 if (typeof duration === 'string') {
45 const split = duration.match(/^([\d\.,]+)\s?(\w+)$/) 47 const split = duration.match(/^([\d.,]+)\s?(\w+)$/)
46 48
47 if (split.length === 3) { 49 if (split.length === 3) {
48 const len = parseFloat(split[1]) 50 const len = parseFloat(split[1])
49 let unit = split[2].replace(/s$/i,'').toLowerCase() 51 let unit = split[2].replace(/s$/i, '').toLowerCase()
50 if (unit === 'm') { 52 if (unit === 'm') {
51 unit = 'ms' 53 unit = 'ms'
52 } 54 }
@@ -73,21 +75,21 @@ export function parseBytes (value: string | number): number {
73 75
74 if (value.match(tgm)) { 76 if (value.match(tgm)) {
75 match = value.match(tgm) 77 match = value.match(tgm)
76 return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 78 return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 +
77 + parseInt(match[2], 10) * 1024 * 1024 * 1024 79 parseInt(match[2], 10) * 1024 * 1024 * 1024 +
78 + parseInt(match[3], 10) * 1024 * 1024 80 parseInt(match[3], 10) * 1024 * 1024
79 } else if (value.match(tg)) { 81 } else if (value.match(tg)) {
80 match = value.match(tg) 82 match = value.match(tg)
81 return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 83 return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 +
82 + parseInt(match[2], 10) * 1024 * 1024 * 1024 84 parseInt(match[2], 10) * 1024 * 1024 * 1024
83 } else if (value.match(tm)) { 85 } else if (value.match(tm)) {
84 match = value.match(tm) 86 match = value.match(tm)
85 return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 87 return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 +
86 + parseInt(match[2], 10) * 1024 * 1024 88 parseInt(match[2], 10) * 1024 * 1024
87 } else if (value.match(gm)) { 89 } else if (value.match(gm)) {
88 match = value.match(gm) 90 match = value.match(gm)
89 return parseInt(match[1], 10) * 1024 * 1024 * 1024 91 return parseInt(match[1], 10) * 1024 * 1024 * 1024 +
90 + parseInt(match[2], 10) * 1024 * 1024 92 parseInt(match[2], 10) * 1024 * 1024
91 } else if (value.match(t)) { 93 } else if (value.match(t)) {
92 match = value.match(t) 94 match = value.match(t)
93 return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 95 return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024
@@ -137,6 +139,7 @@ function getAppNumber () {
137} 139}
138 140
139let rootPath: string 141let rootPath: string
142
140function root () { 143function root () {
141 if (rootPath) return rootPath 144 if (rootPath) return rootPath
142 145
@@ -163,7 +166,7 @@ function escapeHTML (stringParam) {
163 '=': '&#x3D;' 166 '=': '&#x3D;'
164 } 167 }
165 168
166 return String(stringParam).replace(/[&<>"'`=\/]/g, s => entityMap[s]) 169 return String(stringParam).replace(/[&<>"'`=/]/g, s => entityMap[s])
167} 170}
168 171
169function pageToStartAndCount (page: number, itemsPerPage: number) { 172function pageToStartAndCount (page: number, itemsPerPage: number) {
@@ -202,6 +205,7 @@ function sha1 (str: string | Buffer, encoding: HexBase64Latin1Encoding = 'hex')
202function execShell (command: string, options?: ExecOptions) { 205function execShell (command: string, options?: ExecOptions) {
203 return new Promise<{ err?: Error, stdout: string, stderr: string }>((res, rej) => { 206 return new Promise<{ err?: Error, stdout: string, stderr: string }>((res, rej) => {
204 exec(command, options, (err, stdout, stderr) => { 207 exec(command, options, (err, stdout, stderr) => {
208 // eslint-disable-next-line prefer-promise-reject-errors
205 if (err) return rej({ err, stdout, stderr }) 209 if (err) return rej({ err, stdout, stderr })
206 210
207 return res({ stdout, stderr }) 211 return res({ stdout, stderr })
@@ -226,14 +230,6 @@ function promisify1<T, A> (func: (arg: T, cb: (err: any, result: A) => void) =>
226 } 230 }
227} 231}
228 232
229function promisify1WithVoid<T> (func: (arg: T, cb: (err: any) => void) => void): (arg: T) => Promise<void> {
230 return function promisified (arg: T): Promise<void> {
231 return new Promise<void>((resolve: () => void, reject: (err: any) => void) => {
232 func.apply(null, [ arg, (err: any) => err ? reject(err) : resolve() ])
233 })
234 }
235}
236
237function promisify2<T, U, A> (func: (arg1: T, arg2: U, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U) => Promise<A> { 233function promisify2<T, U, A> (func: (arg1: T, arg2: U, cb: (err: any, result: A) => void) => void): (arg1: T, arg2: U) => Promise<A> {
238 return function promisified (arg1: T, arg2: U): Promise<A> { 234 return function promisified (arg1: T, arg2: U): Promise<A> {
239 return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => { 235 return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
@@ -242,15 +238,7 @@ function promisify2<T, U, A> (func: (arg1: T, arg2: U, cb: (err: any, result: A)
242 } 238 }
243} 239}
244 240
245function promisify2WithVoid<T, U> (func: (arg1: T, arg2: U, cb: (err: any) => void) => void): (arg1: T, arg2: U) => Promise<void> { 241const randomBytesPromise = promisify1<number, Buffer>(randomBytes)
246 return function promisified (arg1: T, arg2: U): Promise<void> {
247 return new Promise<void>((resolve: () => void, reject: (err: any) => void) => {
248 func.apply(null, [ arg1, arg2, (err: any) => err ? reject(err) : resolve() ])
249 })
250 }
251}
252
253const pseudoRandomBytesPromise = promisify1<number, Buffer>(pseudoRandomBytes)
254const createPrivateKey = promisify1<number, { key: string }>(pem.createPrivateKey) 242const createPrivateKey = promisify1<number, { key: string }>(pem.createPrivateKey)
255const getPublicKey = promisify1<string, { publicKey: string }>(pem.getPublicKey) 243const getPublicKey = promisify1<string, { publicKey: string }>(pem.getPublicKey)
256const execPromise2 = promisify2<string, any, string>(exec) 244const execPromise2 = promisify2<string, any, string>(exec)
@@ -280,7 +268,7 @@ export {
280 promisify1, 268 promisify1,
281 promisify2, 269 promisify2,
282 270
283 pseudoRandomBytesPromise, 271 randomBytesPromise,
284 createPrivateKey, 272 createPrivateKey,
285 getPublicKey, 273 getPublicKey,
286 execPromise2, 274 execPromise2,
diff --git a/server/helpers/custom-jsonld-signature.ts b/server/helpers/custom-jsonld-signature.ts
index a407a9fec..749c50cb3 100644
--- a/server/helpers/custom-jsonld-signature.ts
+++ b/server/helpers/custom-jsonld-signature.ts
@@ -5,52 +5,52 @@ import { logger } from './logger'
5const CACHE = { 5const CACHE = {
6 'https://w3id.org/security/v1': { 6 'https://w3id.org/security/v1': {
7 '@context': { 7 '@context': {
8 'id': '@id', 8 id: '@id',
9 'type': '@type', 9 type: '@type',
10 10
11 'dc': 'http://purl.org/dc/terms/', 11 dc: 'http://purl.org/dc/terms/',
12 'sec': 'https://w3id.org/security#', 12 sec: 'https://w3id.org/security#',
13 'xsd': 'http://www.w3.org/2001/XMLSchema#', 13 xsd: 'http://www.w3.org/2001/XMLSchema#',
14 14
15 'EcdsaKoblitzSignature2016': 'sec:EcdsaKoblitzSignature2016', 15 EcdsaKoblitzSignature2016: 'sec:EcdsaKoblitzSignature2016',
16 'Ed25519Signature2018': 'sec:Ed25519Signature2018', 16 Ed25519Signature2018: 'sec:Ed25519Signature2018',
17 'EncryptedMessage': 'sec:EncryptedMessage', 17 EncryptedMessage: 'sec:EncryptedMessage',
18 'GraphSignature2012': 'sec:GraphSignature2012', 18 GraphSignature2012: 'sec:GraphSignature2012',
19 'LinkedDataSignature2015': 'sec:LinkedDataSignature2015', 19 LinkedDataSignature2015: 'sec:LinkedDataSignature2015',
20 'LinkedDataSignature2016': 'sec:LinkedDataSignature2016', 20 LinkedDataSignature2016: 'sec:LinkedDataSignature2016',
21 'CryptographicKey': 'sec:Key', 21 CryptographicKey: 'sec:Key',
22 22
23 'authenticationTag': 'sec:authenticationTag', 23 authenticationTag: 'sec:authenticationTag',
24 'canonicalizationAlgorithm': 'sec:canonicalizationAlgorithm', 24 canonicalizationAlgorithm: 'sec:canonicalizationAlgorithm',
25 'cipherAlgorithm': 'sec:cipherAlgorithm', 25 cipherAlgorithm: 'sec:cipherAlgorithm',
26 'cipherData': 'sec:cipherData', 26 cipherData: 'sec:cipherData',
27 'cipherKey': 'sec:cipherKey', 27 cipherKey: 'sec:cipherKey',
28 'created': { '@id': 'dc:created', '@type': 'xsd:dateTime' }, 28 created: { '@id': 'dc:created', '@type': 'xsd:dateTime' },
29 'creator': { '@id': 'dc:creator', '@type': '@id' }, 29 creator: { '@id': 'dc:creator', '@type': '@id' },
30 'digestAlgorithm': 'sec:digestAlgorithm', 30 digestAlgorithm: 'sec:digestAlgorithm',
31 'digestValue': 'sec:digestValue', 31 digestValue: 'sec:digestValue',
32 'domain': 'sec:domain', 32 domain: 'sec:domain',
33 'encryptionKey': 'sec:encryptionKey', 33 encryptionKey: 'sec:encryptionKey',
34 'expiration': { '@id': 'sec:expiration', '@type': 'xsd:dateTime' }, 34 expiration: { '@id': 'sec:expiration', '@type': 'xsd:dateTime' },
35 'expires': { '@id': 'sec:expiration', '@type': 'xsd:dateTime' }, 35 expires: { '@id': 'sec:expiration', '@type': 'xsd:dateTime' },
36 'initializationVector': 'sec:initializationVector', 36 initializationVector: 'sec:initializationVector',
37 'iterationCount': 'sec:iterationCount', 37 iterationCount: 'sec:iterationCount',
38 'nonce': 'sec:nonce', 38 nonce: 'sec:nonce',
39 'normalizationAlgorithm': 'sec:normalizationAlgorithm', 39 normalizationAlgorithm: 'sec:normalizationAlgorithm',
40 'owner': { '@id': 'sec:owner', '@type': '@id' }, 40 owner: { '@id': 'sec:owner', '@type': '@id' },
41 'password': 'sec:password', 41 password: 'sec:password',
42 'privateKey': { '@id': 'sec:privateKey', '@type': '@id' }, 42 privateKey: { '@id': 'sec:privateKey', '@type': '@id' },
43 'privateKeyPem': 'sec:privateKeyPem', 43 privateKeyPem: 'sec:privateKeyPem',
44 'publicKey': { '@id': 'sec:publicKey', '@type': '@id' }, 44 publicKey: { '@id': 'sec:publicKey', '@type': '@id' },
45 'publicKeyBase58': 'sec:publicKeyBase58', 45 publicKeyBase58: 'sec:publicKeyBase58',
46 'publicKeyPem': 'sec:publicKeyPem', 46 publicKeyPem: 'sec:publicKeyPem',
47 'publicKeyWif': 'sec:publicKeyWif', 47 publicKeyWif: 'sec:publicKeyWif',
48 'publicKeyService': { '@id': 'sec:publicKeyService', '@type': '@id' }, 48 publicKeyService: { '@id': 'sec:publicKeyService', '@type': '@id' },
49 'revoked': { '@id': 'sec:revoked', '@type': 'xsd:dateTime' }, 49 revoked: { '@id': 'sec:revoked', '@type': 'xsd:dateTime' },
50 'salt': 'sec:salt', 50 salt: 'sec:salt',
51 'signature': 'sec:signature', 51 signature: 'sec:signature',
52 'signatureAlgorithm': 'sec:signingAlgorithm', 52 signatureAlgorithm: 'sec:signingAlgorithm',
53 'signatureValue': 'sec:signatureValue' 53 signatureValue: 'sec:signatureValue'
54 } 54 }
55 } 55 }
56} 56}
@@ -60,12 +60,12 @@ const nodeDocumentLoader = jsonld.documentLoaders.node()
60const lru = new AsyncLRU({ 60const lru = new AsyncLRU({
61 max: 10, 61 max: 10,
62 load: (url, cb) => { 62 load: (url, cb) => {
63 if (CACHE[ url ] !== undefined) { 63 if (CACHE[url] !== undefined) {
64 logger.debug('Using cache for JSON-LD %s.', url) 64 logger.debug('Using cache for JSON-LD %s.', url)
65 65
66 return cb(null, { 66 return cb(null, {
67 contextUrl: null, 67 contextUrl: null,
68 document: CACHE[ url ], 68 document: CACHE[url],
69 documentUrl: url 69 documentUrl: url
70 }) 70 })
71 } 71 }
diff --git a/server/helpers/custom-validators/activitypub/actor.ts b/server/helpers/custom-validators/activitypub/actor.ts
index fa58e163f..2f44522a5 100644
--- a/server/helpers/custom-validators/activitypub/actor.ts
+++ b/server/helpers/custom-validators/activitypub/actor.ts
@@ -6,7 +6,7 @@ import { isHostValid } from '../servers'
6import { peertubeTruncate } from '@server/helpers/core-utils' 6import { peertubeTruncate } from '@server/helpers/core-utils'
7 7
8function isActorEndpointsObjectValid (endpointObject: any) { 8function isActorEndpointsObjectValid (endpointObject: any) {
9 if (endpointObject && endpointObject.sharedInbox) { 9 if (endpointObject?.sharedInbox) {
10 return isActivityPubUrlValid(endpointObject.sharedInbox) 10 return isActivityPubUrlValid(endpointObject.sharedInbox)
11 } 11 }
12 12
@@ -28,7 +28,7 @@ function isActorPublicKeyValid (publicKey: string) {
28 return exists(publicKey) && 28 return exists(publicKey) &&
29 typeof publicKey === 'string' && 29 typeof publicKey === 'string' &&
30 publicKey.startsWith('-----BEGIN PUBLIC KEY-----') && 30 publicKey.startsWith('-----BEGIN PUBLIC KEY-----') &&
31 publicKey.indexOf('-----END PUBLIC KEY-----') !== -1 && 31 publicKey.includes('-----END PUBLIC KEY-----') &&
32 validator.isLength(publicKey, CONSTRAINTS_FIELDS.ACTORS.PUBLIC_KEY) 32 validator.isLength(publicKey, CONSTRAINTS_FIELDS.ACTORS.PUBLIC_KEY)
33} 33}
34 34
@@ -43,7 +43,7 @@ function isActorPrivateKeyValid (privateKey: string) {
43 typeof privateKey === 'string' && 43 typeof privateKey === 'string' &&
44 privateKey.startsWith('-----BEGIN RSA PRIVATE KEY-----') && 44 privateKey.startsWith('-----BEGIN RSA PRIVATE KEY-----') &&
45 // Sometimes there is a \n at the end, so just assert the string contains the end mark 45 // Sometimes there is a \n at the end, so just assert the string contains the end mark
46 privateKey.indexOf('-----END RSA PRIVATE KEY-----') !== -1 && 46 privateKey.includes('-----END RSA PRIVATE KEY-----') &&
47 validator.isLength(privateKey, CONSTRAINTS_FIELDS.ACTORS.PRIVATE_KEY) 47 validator.isLength(privateKey, CONSTRAINTS_FIELDS.ACTORS.PRIVATE_KEY)
48} 48}
49 49
@@ -101,8 +101,6 @@ function normalizeActor (actor: any) {
101 actor.summary = null 101 actor.summary = null
102 } 102 }
103 } 103 }
104
105 return
106} 104}
107 105
108function isValidActorHandle (handle: string) { 106function isValidActorHandle (handle: string) {
diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts
index 21d5c53ca..c5b3b4d9f 100644
--- a/server/helpers/custom-validators/activitypub/cache-file.ts
+++ b/server/helpers/custom-validators/activitypub/cache-file.ts
@@ -6,7 +6,7 @@ import { CacheFileObject } from '../../../../shared/models/activitypub/objects'
6function isCacheFileObjectValid (object: CacheFileObject) { 6function isCacheFileObjectValid (object: CacheFileObject) {
7 return exists(object) && 7 return exists(object) &&
8 object.type === 'CacheFile' && 8 object.type === 'CacheFile' &&
9 isDateValid(object.expires) && 9 (object.expires === null || isDateValid(object.expires)) &&
10 isActivityPubUrlValid(object.object) && 10 isActivityPubUrlValid(object.object) &&
11 (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url)) 11 (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url))
12} 12}
diff --git a/server/helpers/custom-validators/activitypub/video-comments.ts b/server/helpers/custom-validators/activitypub/video-comments.ts
index aa3c246b5..ea852c491 100644
--- a/server/helpers/custom-validators/activitypub/video-comments.ts
+++ b/server/helpers/custom-validators/activitypub/video-comments.ts
@@ -48,8 +48,6 @@ function normalizeComment (comment: any) {
48 if (typeof comment.url === 'object') comment.url = comment.url.href || comment.url.url 48 if (typeof comment.url === 'object') comment.url = comment.url.href || comment.url.url
49 else comment.url = comment.id 49 else comment.url = comment.id
50 } 50 }
51
52 return
53} 51}
54 52
55function isCommentTypeValid (comment: any): boolean { 53function isCommentTypeValid (comment: any): boolean {
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index fe94bd58a..876cc7f50 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -13,6 +13,7 @@ import {
13import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc' 13import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc'
14import { VideoState } from '../../../../shared/models/videos' 14import { VideoState } from '../../../../shared/models/videos'
15import { logger } from '@server/helpers/logger' 15import { logger } from '@server/helpers/logger'
16import { ActivityVideoFileMetadataObject } from '@shared/models'
16 17
17function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { 18function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) {
18 return isBaseActivityValid(activity, 'Update') && 19 return isBaseActivityValid(activity, 'Update') &&
@@ -51,11 +52,16 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
51 logger.debug('Video has invalid captions', { video }) 52 logger.debug('Video has invalid captions', { video })
52 return false 53 return false
53 } 54 }
55 if (!setValidRemoteIcon(video)) {
56 logger.debug('Video has invalid icons', { video })
57 return false
58 }
54 59
55 // Default attributes 60 // Default attributes
56 if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED 61 if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED
57 if (!isBooleanValid(video.waitTranscoding)) video.waitTranscoding = false 62 if (!isBooleanValid(video.waitTranscoding)) video.waitTranscoding = false
58 if (!isBooleanValid(video.downloadEnabled)) video.downloadEnabled = true 63 if (!isBooleanValid(video.downloadEnabled)) video.downloadEnabled = true
64 if (!isBooleanValid(video.commentsEnabled)) video.commentsEnabled = false
59 65
60 return isActivityPubUrlValid(video.id) && 66 return isActivityPubUrlValid(video.id) &&
61 isVideoNameValid(video.name) && 67 isVideoNameValid(video.name) &&
@@ -72,7 +78,6 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
72 isDateValid(video.updated) && 78 isDateValid(video.updated) &&
73 (!video.originallyPublishedAt || isDateValid(video.originallyPublishedAt)) && 79 (!video.originallyPublishedAt || isDateValid(video.originallyPublishedAt)) &&
74 (!video.content || isRemoteVideoContentValid(video.mediaType, video.content)) && 80 (!video.content || isRemoteVideoContentValid(video.mediaType, video.content)) &&
75 isRemoteVideoIconValid(video.icon) &&
76 video.url.length !== 0 && 81 video.url.length !== 0 &&
77 video.attributedTo.length !== 0 82 video.attributedTo.length !== 0
78} 83}
@@ -80,19 +85,19 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
80function isRemoteVideoUrlValid (url: any) { 85function isRemoteVideoUrlValid (url: any) {
81 return url.type === 'Link' && 86 return url.type === 'Link' &&
82 ( 87 (
83 ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mediaType) !== -1 && 88 ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.includes(url.mediaType) &&
84 isActivityPubUrlValid(url.href) && 89 isActivityPubUrlValid(url.href) &&
85 validator.isInt(url.height + '', { min: 0 }) && 90 validator.isInt(url.height + '', { min: 0 }) &&
86 validator.isInt(url.size + '', { min: 0 }) && 91 validator.isInt(url.size + '', { min: 0 }) &&
87 (!url.fps || validator.isInt(url.fps + '', { min: -1 })) 92 (!url.fps || validator.isInt(url.fps + '', { min: -1 }))
88 ) || 93 ) ||
89 ( 94 (
90 ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mediaType) !== -1 && 95 ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.includes(url.mediaType) &&
91 isActivityPubUrlValid(url.href) && 96 isActivityPubUrlValid(url.href) &&
92 validator.isInt(url.height + '', { min: 0 }) 97 validator.isInt(url.height + '', { min: 0 })
93 ) || 98 ) ||
94 ( 99 (
95 ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mediaType) !== -1 && 100 ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.includes(url.mediaType) &&
96 validator.isLength(url.href, { min: 5 }) && 101 validator.isLength(url.href, { min: 5 }) &&
97 validator.isInt(url.height + '', { min: 0 }) 102 validator.isInt(url.height + '', { min: 0 })
98 ) || 103 ) ||
@@ -100,7 +105,15 @@ function isRemoteVideoUrlValid (url: any) {
100 (url.mediaType || url.mimeType) === 'application/x-mpegURL' && 105 (url.mediaType || url.mimeType) === 'application/x-mpegURL' &&
101 isActivityPubUrlValid(url.href) && 106 isActivityPubUrlValid(url.href) &&
102 isArray(url.tag) 107 isArray(url.tag)
103 ) 108 ) ||
109 isAPVideoFileMetadataObject(url)
110}
111
112function isAPVideoFileMetadataObject (url: any): url is ActivityVideoFileMetadataObject {
113 return url &&
114 url.type === 'Link' &&
115 url.mediaType === 'application/json' &&
116 isArray(url.rel) && url.rel.includes('metadata')
104} 117}
105 118
106// --------------------------------------------------------------------------- 119// ---------------------------------------------------------------------------
@@ -109,7 +122,8 @@ export {
109 sanitizeAndCheckVideoTorrentUpdateActivity, 122 sanitizeAndCheckVideoTorrentUpdateActivity,
110 isRemoteStringIdentifierValid, 123 isRemoteStringIdentifierValid,
111 sanitizeAndCheckVideoTorrentObject, 124 sanitizeAndCheckVideoTorrentObject,
112 isRemoteVideoUrlValid 125 isRemoteVideoUrlValid,
126 isAPVideoFileMetadataObject
113} 127}
114 128
115// --------------------------------------------------------------------------- 129// ---------------------------------------------------------------------------
@@ -131,6 +145,8 @@ function setValidRemoteCaptions (video: any) {
131 if (Array.isArray(video.subtitleLanguage) === false) return false 145 if (Array.isArray(video.subtitleLanguage) === false) return false
132 146
133 video.subtitleLanguage = video.subtitleLanguage.filter(caption => { 147 video.subtitleLanguage = video.subtitleLanguage.filter(caption => {
148 if (!isActivityPubUrlValid(caption.url)) caption.url = null
149
134 return isRemoteStringIdentifierValid(caption) 150 return isRemoteStringIdentifierValid(caption)
135 }) 151 })
136 152
@@ -149,12 +165,19 @@ function isRemoteVideoContentValid (mediaType: string, content: string) {
149 return mediaType === 'text/markdown' && isVideoTruncatedDescriptionValid(content) 165 return mediaType === 'text/markdown' && isVideoTruncatedDescriptionValid(content)
150} 166}
151 167
152function isRemoteVideoIconValid (icon: any) { 168function setValidRemoteIcon (video: any) {
153 return icon.type === 'Image' && 169 if (video.icon && !isArray(video.icon)) video.icon = [ video.icon ]
154 isActivityPubUrlValid(icon.url) && 170 if (!video.icon) video.icon = []
155 icon.mediaType === 'image/jpeg' && 171
156 validator.isInt(icon.width + '', { min: 0 }) && 172 video.icon = video.icon.filter(icon => {
157 validator.isInt(icon.height + '', { min: 0 }) 173 return icon.type === 'Image' &&
174 isActivityPubUrlValid(icon.url) &&
175 icon.mediaType === 'image/jpeg' &&
176 validator.isInt(icon.width + '', { min: 0 }) &&
177 validator.isInt(icon.height + '', { min: 0 })
178 })
179
180 return video.icon.length !== 0
158} 181}
159 182
160function setValidRemoteVideoUrls (video: any) { 183function setValidRemoteVideoUrls (video: any) {
diff --git a/server/helpers/custom-validators/feeds.ts b/server/helpers/custom-validators/feeds.ts
index 638e814f0..fa35a7da6 100644
--- a/server/helpers/custom-validators/feeds.ts
+++ b/server/helpers/custom-validators/feeds.ts
@@ -13,7 +13,7 @@ function isValidRSSFeed (value: string) {
13 'atom1' 13 'atom1'
14 ] 14 ]
15 15
16 return feedExtensions.indexOf(value) !== -1 16 return feedExtensions.includes(value)
17} 17}
18 18
19// --------------------------------------------------------------------------- 19// ---------------------------------------------------------------------------
diff --git a/server/helpers/custom-validators/logs.ts b/server/helpers/custom-validators/logs.ts
index 30d0ce262..0f266ed3b 100644
--- a/server/helpers/custom-validators/logs.ts
+++ b/server/helpers/custom-validators/logs.ts
@@ -4,7 +4,7 @@ import { LogLevel } from '../../../shared/models/server/log-level.type'
4const logLevels: LogLevel[] = [ 'debug', 'info', 'warn', 'error' ] 4const logLevels: LogLevel[] = [ 'debug', 'info', 'warn', 'error' ]
5 5
6function isValidLogLevel (value: any) { 6function isValidLogLevel (value: any) {
7 return exists(value) && logLevels.indexOf(value) !== -1 7 return exists(value) && logLevels.includes(value)
8} 8}
9 9
10// --------------------------------------------------------------------------- 10// ---------------------------------------------------------------------------
diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts
index 89149b3e0..cf32201c4 100644
--- a/server/helpers/custom-validators/misc.ts
+++ b/server/helpers/custom-validators/misc.ts
@@ -94,13 +94,13 @@ function isFileValid (
94 if (isArray(files)) return optional 94 if (isArray(files)) return optional
95 95
96 // Should have a file 96 // Should have a file
97 const fileArray = files[ field ] 97 const fileArray = files[field]
98 if (!fileArray || fileArray.length === 0) { 98 if (!fileArray || fileArray.length === 0) {
99 return optional 99 return optional
100 } 100 }
101 101
102 // The file should exist 102 // The file should exist
103 const file = fileArray[ 0 ] 103 const file = fileArray[0]
104 if (!file || !file.originalname) return false 104 if (!file || !file.originalname) return false
105 105
106 // Check size 106 // Check size
diff --git a/server/helpers/custom-validators/plugins.ts b/server/helpers/custom-validators/plugins.ts
index 3af72547b..d2fc03936 100644
--- a/server/helpers/custom-validators/plugins.ts
+++ b/server/helpers/custom-validators/plugins.ts
@@ -14,7 +14,7 @@ function isPluginTypeValid (value: any) {
14function isPluginNameValid (value: string) { 14function isPluginNameValid (value: string) {
15 return exists(value) && 15 return exists(value) &&
16 validator.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.NAME) && 16 validator.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.NAME) &&
17 validator.matches(value, /^[a-z\-]+$/) 17 validator.matches(value, /^[a-z-0-9]+$/)
18} 18}
19 19
20function isNpmPluginNameValid (value: string) { 20function isNpmPluginNameValid (value: string) {
@@ -146,8 +146,8 @@ function isPackageJSONValid (packageJSON: PluginPackageJson, pluginType: PluginT
146} 146}
147 147
148function isLibraryCodeValid (library: any) { 148function isLibraryCodeValid (library: any) {
149 return typeof library.register === 'function' 149 return typeof library.register === 'function' &&
150 && typeof library.unregister === 'function' 150 typeof library.unregister === 'function'
151} 151}
152 152
153export { 153export {
diff --git a/server/helpers/custom-validators/user-notifications.ts b/server/helpers/custom-validators/user-notifications.ts
index 5a4d10504..8a33b895b 100644
--- a/server/helpers/custom-validators/user-notifications.ts
+++ b/server/helpers/custom-validators/user-notifications.ts
@@ -9,7 +9,8 @@ function isUserNotificationTypeValid (value: any) {
9 9
10function isUserNotificationSettingValid (value: any) { 10function isUserNotificationSettingValid (value: any) {
11 return exists(value) && 11 return exists(value) &&
12 validator.isInt('' + value) && ( 12 validator.isInt('' + value) &&
13 (
13 value === UserNotificationSettingValue.NONE || 14 value === UserNotificationSettingValue.NONE ||
14 value === UserNotificationSettingValue.WEB || 15 value === UserNotificationSettingValue.WEB ||
15 value === UserNotificationSettingValue.EMAIL || 16 value === UserNotificationSettingValue.EMAIL ||
diff --git a/server/helpers/custom-validators/users.ts b/server/helpers/custom-validators/users.ts
index b4d5751e7..d6e91ad35 100644
--- a/server/helpers/custom-validators/users.ts
+++ b/server/helpers/custom-validators/users.ts
@@ -3,6 +3,7 @@ import { UserRole } from '../../../shared'
3import { CONSTRAINTS_FIELDS, NSFW_POLICY_TYPES } from '../../initializers/constants' 3import { CONSTRAINTS_FIELDS, NSFW_POLICY_TYPES } from '../../initializers/constants'
4import { exists, isArray, isBooleanValid, isFileValid } from './misc' 4import { exists, isArray, isBooleanValid, isFileValid } from './misc'
5import { values } from 'lodash' 5import { values } from 'lodash'
6import { isEmailEnabled } from '../../initializers/config'
6 7
7const USERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.USERS 8const USERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.USERS
8 9
@@ -10,6 +11,13 @@ function isUserPasswordValid (value: string) {
10 return validator.isLength(value, USERS_CONSTRAINTS_FIELDS.PASSWORD) 11 return validator.isLength(value, USERS_CONSTRAINTS_FIELDS.PASSWORD)
11} 12}
12 13
14function isUserPasswordValidOrEmpty (value: string) {
15 // Empty password is only possible if emailing is enabled.
16 if (value === '') return isEmailEnabled()
17
18 return isUserPasswordValid(value)
19}
20
13function isUserVideoQuotaValid (value: string) { 21function isUserVideoQuotaValid (value: string) {
14 return exists(value) && validator.isInt(value + '', USERS_CONSTRAINTS_FIELDS.VIDEO_QUOTA) 22 return exists(value) && validator.isInt(value + '', USERS_CONSTRAINTS_FIELDS.VIDEO_QUOTA)
15} 23}
@@ -38,7 +46,7 @@ function isUserEmailVerifiedValid (value: any) {
38 46
39const nsfwPolicies = values(NSFW_POLICY_TYPES) 47const nsfwPolicies = values(NSFW_POLICY_TYPES)
40function isUserNSFWPolicyValid (value: any) { 48function isUserNSFWPolicyValid (value: any) {
41 return exists(value) && nsfwPolicies.indexOf(value) !== -1 49 return exists(value) && nsfwPolicies.includes(value)
42} 50}
43 51
44function isUserWebTorrentEnabledValid (value: any) { 52function isUserWebTorrentEnabledValid (value: any) {
@@ -103,6 +111,7 @@ export {
103 isUserVideosHistoryEnabledValid, 111 isUserVideosHistoryEnabledValid,
104 isUserBlockedValid, 112 isUserBlockedValid,
105 isUserPasswordValid, 113 isUserPasswordValid,
114 isUserPasswordValidOrEmpty,
106 isUserVideoLanguages, 115 isUserVideoLanguages,
107 isUserBlockedReasonValid, 116 isUserBlockedReasonValid,
108 isUserRoleValid, 117 isUserRoleValid,
diff --git a/server/helpers/custom-validators/video-abuses.ts b/server/helpers/custom-validators/video-abuses.ts
index a9478c76a..05e11b1c6 100644
--- a/server/helpers/custom-validators/video-abuses.ts
+++ b/server/helpers/custom-validators/video-abuses.ts
@@ -1,8 +1,8 @@
1import { Response } from 'express'
2import validator from 'validator' 1import validator from 'validator'
2
3import { CONSTRAINTS_FIELDS, VIDEO_ABUSE_STATES } from '../../initializers/constants' 3import { CONSTRAINTS_FIELDS, VIDEO_ABUSE_STATES } from '../../initializers/constants'
4import { exists } from './misc' 4import { exists } from './misc'
5import { VideoAbuseModel } from '../../models/video/video-abuse' 5import { VideoAbuseVideoIs } from '@shared/models/videos/abuse/video-abuse-video-is.type'
6 6
7const VIDEO_ABUSES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_ABUSES 7const VIDEO_ABUSES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_ABUSES
8 8
@@ -15,7 +15,14 @@ function isVideoAbuseModerationCommentValid (value: string) {
15} 15}
16 16
17function isVideoAbuseStateValid (value: string) { 17function isVideoAbuseStateValid (value: string) {
18 return exists(value) && VIDEO_ABUSE_STATES[ value ] !== undefined 18 return exists(value) && VIDEO_ABUSE_STATES[value] !== undefined
19}
20
21function isAbuseVideoIsValid (value: VideoAbuseVideoIs) {
22 return exists(value) && (
23 value === 'deleted' ||
24 value === 'blacklisted'
25 )
19} 26}
20 27
21// --------------------------------------------------------------------------- 28// ---------------------------------------------------------------------------
@@ -23,5 +30,6 @@ function isVideoAbuseStateValid (value: string) {
23export { 30export {
24 isVideoAbuseStateValid, 31 isVideoAbuseStateValid,
25 isVideoAbuseReasonValid, 32 isVideoAbuseReasonValid,
33 isAbuseVideoIsValid,
26 isVideoAbuseModerationCommentValid 34 isVideoAbuseModerationCommentValid
27} 35}
diff --git a/server/helpers/custom-validators/video-captions.ts b/server/helpers/custom-validators/video-captions.ts
index d06eb3695..528edf60c 100644
--- a/server/helpers/custom-validators/video-captions.ts
+++ b/server/helpers/custom-validators/video-captions.ts
@@ -2,13 +2,13 @@ import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_LANGUAGES } from '../../initialize
2import { exists, isFileValid } from './misc' 2import { exists, isFileValid } from './misc'
3 3
4function isVideoCaptionLanguageValid (value: any) { 4function isVideoCaptionLanguageValid (value: any) {
5 return exists(value) && VIDEO_LANGUAGES[ value ] !== undefined 5 return exists(value) && VIDEO_LANGUAGES[value] !== undefined
6} 6}
7 7
8const videoCaptionTypes = Object.keys(MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT) 8const videoCaptionTypesRegex = Object.keys(MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT)
9 .concat([ 'application/octet-stream' ]) // MacOS sends application/octet-stream >< 9 .concat([ 'application/octet-stream' ]) // MacOS sends application/octet-stream
10 .map(m => `(${m})`) 10 .map(m => `(${m})`)
11const videoCaptionTypesRegex = videoCaptionTypes.join('|') 11 .join('|')
12function isVideoCaptionFile (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[], field: string) { 12function isVideoCaptionFile (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[], field: string) {
13 return isFileValid(files, videoCaptionTypesRegex, field, CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max) 13 return isFileValid(files, videoCaptionTypesRegex, field, CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max)
14} 14}
diff --git a/server/helpers/custom-validators/video-imports.ts b/server/helpers/custom-validators/video-imports.ts
index ffad482b4..33a1fa8ab 100644
--- a/server/helpers/custom-validators/video-imports.ts
+++ b/server/helpers/custom-validators/video-imports.ts
@@ -20,11 +20,13 @@ function isVideoImportTargetUrlValid (url: string) {
20} 20}
21 21
22function isVideoImportStateValid (value: any) { 22function isVideoImportStateValid (value: any) {
23 return exists(value) && VIDEO_IMPORT_STATES[ value ] !== undefined 23 return exists(value) && VIDEO_IMPORT_STATES[value] !== undefined
24} 24}
25 25
26const videoTorrentImportTypes = Object.keys(MIMETYPES.TORRENT.MIMETYPE_EXT).map(m => `(${m})`) 26const videoTorrentImportRegex = Object.keys(MIMETYPES.TORRENT.MIMETYPE_EXT)
27const videoTorrentImportRegex = videoTorrentImportTypes.join('|') 27 .concat([ 'application/octet-stream' ]) // MacOS sends application/octet-stream
28 .map(m => `(${m})`)
29 .join('|')
28function isVideoImportTorrentFile (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) { 30function isVideoImportTorrentFile (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) {
29 return isFileValid(files, videoTorrentImportRegex, 'torrentfile', CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_FILE.FILE_SIZE.max, true) 31 return isFileValid(files, videoTorrentImportRegex, 'torrentfile', CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_FILE.FILE_SIZE.max, true)
30} 32}
diff --git a/server/helpers/custom-validators/video-playlists.ts b/server/helpers/custom-validators/video-playlists.ts
index 4bb8384ab..180018fc5 100644
--- a/server/helpers/custom-validators/video-playlists.ts
+++ b/server/helpers/custom-validators/video-playlists.ts
@@ -1,8 +1,6 @@
1import { exists } from './misc' 1import { exists } from './misc'
2import validator from 'validator' 2import validator from 'validator'
3import { CONSTRAINTS_FIELDS, VIDEO_PLAYLIST_PRIVACIES, VIDEO_PLAYLIST_TYPES } from '../../initializers/constants' 3import { CONSTRAINTS_FIELDS, VIDEO_PLAYLIST_PRIVACIES, VIDEO_PLAYLIST_TYPES } from '../../initializers/constants'
4import * as express from 'express'
5import { VideoPlaylistModel } from '../../models/video/video-playlist'
6 4
7const PLAYLISTS_CONSTRAINT_FIELDS = CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS 5const PLAYLISTS_CONSTRAINT_FIELDS = CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS
8 6
@@ -15,7 +13,7 @@ function isVideoPlaylistDescriptionValid (value: any) {
15} 13}
16 14
17function isVideoPlaylistPrivacyValid (value: number) { 15function isVideoPlaylistPrivacyValid (value: number) {
18 return validator.isInt(value + '') && VIDEO_PLAYLIST_PRIVACIES[ value ] !== undefined 16 return validator.isInt(value + '') && VIDEO_PLAYLIST_PRIVACIES[value] !== undefined
19} 17}
20 18
21function isVideoPlaylistTimestampValid (value: any) { 19function isVideoPlaylistTimestampValid (value: any) {
@@ -23,7 +21,7 @@ function isVideoPlaylistTimestampValid (value: any) {
23} 21}
24 22
25function isVideoPlaylistTypeValid (value: any) { 23function isVideoPlaylistTypeValid (value: any) {
26 return exists(value) && VIDEO_PLAYLIST_TYPES[ value ] !== undefined 24 return exists(value) && VIDEO_PLAYLIST_TYPES[value] !== undefined
27} 25}
28 26
29// --------------------------------------------------------------------------- 27// ---------------------------------------------------------------------------
diff --git a/server/helpers/custom-validators/video-redundancies.ts b/server/helpers/custom-validators/video-redundancies.ts
new file mode 100644
index 000000000..50a559c4f
--- /dev/null
+++ b/server/helpers/custom-validators/video-redundancies.ts
@@ -0,0 +1,12 @@
1import { exists } from './misc'
2
3function isVideoRedundancyTarget (value: any) {
4 return exists(value) &&
5 (value === 'my-videos' || value === 'remote-videos')
6}
7
8// ---------------------------------------------------------------------------
9
10export {
11 isVideoRedundancyTarget
12}
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts
index a9e859e54..60e8075f6 100644
--- a/server/helpers/custom-validators/videos.ts
+++ b/server/helpers/custom-validators/videos.ts
@@ -20,15 +20,15 @@ function isVideoFilterValid (filter: VideoFilter) {
20} 20}
21 21
22function isVideoCategoryValid (value: any) { 22function isVideoCategoryValid (value: any) {
23 return value === null || VIDEO_CATEGORIES[ value ] !== undefined 23 return value === null || VIDEO_CATEGORIES[value] !== undefined
24} 24}
25 25
26function isVideoStateValid (value: any) { 26function isVideoStateValid (value: any) {
27 return exists(value) && VIDEO_STATES[ value ] !== undefined 27 return exists(value) && VIDEO_STATES[value] !== undefined
28} 28}
29 29
30function isVideoLicenceValid (value: any) { 30function isVideoLicenceValid (value: any) {
31 return value === null || VIDEO_LICENCES[ value ] !== undefined 31 return value === null || VIDEO_LICENCES[value] !== undefined
32} 32}
33 33
34function isVideoLanguageValid (value: any) { 34function isVideoLanguageValid (value: any) {
@@ -73,7 +73,7 @@ function isVideoViewsValid (value: string) {
73} 73}
74 74
75function isVideoRatingTypeValid (value: string) { 75function isVideoRatingTypeValid (value: string) {
76 return value === 'none' || values(VIDEO_RATE_TYPES).indexOf(value as VideoRateType) !== -1 76 return value === 'none' || values(VIDEO_RATE_TYPES).includes(value as VideoRateType)
77} 77}
78 78
79function isVideoFileExtnameValid (value: string) { 79function isVideoFileExtnameValid (value: string) {
@@ -98,7 +98,7 @@ function isVideoImage (files: { [ fieldname: string ]: Express.Multer.File[] } |
98} 98}
99 99
100function isVideoPrivacyValid (value: number) { 100function isVideoPrivacyValid (value: number) {
101 return VIDEO_PRIVACIES[ value ] !== undefined 101 return VIDEO_PRIVACIES[value] !== undefined
102} 102}
103 103
104function isScheduleVideoUpdatePrivacyValid (value: number) { 104function isScheduleVideoUpdatePrivacyValid (value: number) {
diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts
index 9bf6d85a8..f46812977 100644
--- a/server/helpers/express-utils.ts
+++ b/server/helpers/express-utils.ts
@@ -12,7 +12,7 @@ function buildNSFWFilter (res?: express.Response, paramNSFW?: string) {
12 if (paramNSFW === 'false') return false 12 if (paramNSFW === 'false') return false
13 if (paramNSFW === 'both') return undefined 13 if (paramNSFW === 'both') return undefined
14 14
15 if (res && res.locals.oauth) { 15 if (res?.locals.oauth) {
16 const user = res.locals.oauth.token.User 16 const user = res.locals.oauth.token.User
17 17
18 // User does not want NSFW videos 18 // User does not want NSFW videos
@@ -28,7 +28,7 @@ function buildNSFWFilter (res?: express.Response, paramNSFW?: string) {
28 return null 28 return null
29} 29}
30 30
31function cleanUpReqFiles (req: { files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[] }) { 31function cleanUpReqFiles (req: { files: { [fieldname: string]: Express.Multer.File[] } | Express.Multer.File[] }) {
32 const files = req.files 32 const files = req.files
33 33
34 if (!files) return 34 if (!files) return
@@ -39,7 +39,7 @@ function cleanUpReqFiles (req: { files: { [ fieldname: string ]: Express.Multer.
39 } 39 }
40 40
41 for (const key of Object.keys(files)) { 41 for (const key of Object.keys(files)) {
42 const file = files[ key ] 42 const file = files[key]
43 43
44 if (isArray(file)) file.forEach(f => deleteFileAsync(f.path)) 44 if (isArray(file)) file.forEach(f => deleteFileAsync(f.path))
45 else deleteFileAsync(file.path) 45 else deleteFileAsync(file.path)
@@ -65,18 +65,18 @@ function badRequest (req: express.Request, res: express.Response) {
65 65
66function createReqFiles ( 66function createReqFiles (
67 fieldNames: string[], 67 fieldNames: string[],
68 mimeTypes: { [ id: string ]: string }, 68 mimeTypes: { [id: string]: string },
69 destinations: { [ fieldName: string ]: string } 69 destinations: { [fieldName: string]: string }
70) { 70) {
71 const storage = multer.diskStorage({ 71 const storage = multer.diskStorage({
72 destination: (req, file, cb) => { 72 destination: (req, file, cb) => {
73 cb(null, destinations[ file.fieldname ]) 73 cb(null, destinations[file.fieldname])
74 }, 74 },
75 75
76 filename: async (req, file, cb) => { 76 filename: async (req, file, cb) => {
77 let extension: string 77 let extension: string
78 const fileExtension = extname(file.originalname) 78 const fileExtension = extname(file.originalname)
79 const extensionFromMimetype = mimeTypes[ file.mimetype ] 79 const extensionFromMimetype = mimeTypes[file.mimetype]
80 80
81 // Take the file extension if we don't understand the mime type 81 // Take the file extension if we don't understand the mime type
82 // We have the OGG/OGV exception too because firefox sends a bad mime type when sending an OGG file 82 // We have the OGG/OGV exception too because firefox sends a bad mime type when sending an OGG file
@@ -99,7 +99,7 @@ function createReqFiles (
99 } 99 }
100 }) 100 })
101 101
102 let fields: { name: string, maxCount: number }[] = [] 102 const fields: { name: string, maxCount: number }[] = []
103 for (const fieldName of fieldNames) { 103 for (const fieldName of fieldNames) {
104 fields.push({ 104 fields.push({
105 name: fieldName, 105 name: fieldName,
diff --git a/server/helpers/ffmpeg-utils.ts b/server/helpers/ffmpeg-utils.ts
index 00c32e99a..557fb5e3a 100644
--- a/server/helpers/ffmpeg-utils.ts
+++ b/server/helpers/ffmpeg-utils.ts
@@ -1,12 +1,78 @@
1import * as ffmpeg from 'fluent-ffmpeg' 1import * as ffmpeg from 'fluent-ffmpeg'
2import { dirname, join } from 'path' 2import { dirname, join } from 'path'
3import { getTargetBitrate, getMaxBitrate, VideoResolution } from '../../shared/models/videos' 3import { getMaxBitrate, getTargetBitrate, VideoResolution } from '../../shared/models/videos'
4import { FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants' 4import { FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers/constants'
5import { processImage } from './image-utils' 5import { processImage } from './image-utils'
6import { logger } from './logger' 6import { logger } from './logger'
7import { checkFFmpegEncoders } from '../initializers/checker-before-init' 7import { checkFFmpegEncoders } from '../initializers/checker-before-init'
8import { readFile, remove, writeFile } from 'fs-extra' 8import { readFile, remove, writeFile } from 'fs-extra'
9import { CONFIG } from '../initializers/config' 9import { CONFIG } from '../initializers/config'
10import { VideoFileMetadata } from '@shared/models/videos/video-file-metadata'
11
12/**
13 * A toolbox to play with audio
14 */
15namespace audio {
16 export const get = (videoPath: string) => {
17 // without position, ffprobe considers the last input only
18 // we make it consider the first input only
19 // if you pass a file path to pos, then ffprobe acts on that file directly
20 return new Promise<{ absolutePath: string, audioStream?: any }>((res, rej) => {
21
22 function parseFfprobe (err: any, data: ffmpeg.FfprobeData) {
23 if (err) return rej(err)
24
25 if ('streams' in data) {
26 const audioStream = data.streams.find(stream => stream['codec_type'] === 'audio')
27 if (audioStream) {
28 return res({
29 absolutePath: data.format.filename,
30 audioStream
31 })
32 }
33 }
34
35 return res({ absolutePath: data.format.filename })
36 }
37
38 return ffmpeg.ffprobe(videoPath, parseFfprobe)
39 })
40 }
41
42 export namespace bitrate {
43 const baseKbitrate = 384
44
45 const toBits = (kbits: number) => kbits * 8000
46
47 export const aac = (bitrate: number): number => {
48 switch (true) {
49 case bitrate > toBits(baseKbitrate):
50 return baseKbitrate
51
52 default:
53 return -1 // we interpret it as a signal to copy the audio stream as is
54 }
55 }
56
57 export const mp3 = (bitrate: number): number => {
58 /*
59 a 192kbit/sec mp3 doesn't hold as much information as a 192kbit/sec aac.
60 That's why, when using aac, we can go to lower kbit/sec. The equivalences
61 made here are not made to be accurate, especially with good mp3 encoders.
62 */
63 switch (true) {
64 case bitrate <= toBits(192):
65 return 128
66
67 case bitrate <= toBits(384):
68 return 256
69
70 default:
71 return baseKbitrate
72 }
73 }
74 }
75}
10 76
11function computeResolutionsToTranscode (videoFileHeight: number) { 77function computeResolutionsToTranscode (videoFileHeight: number) {
12 const resolutionsEnabled: number[] = [] 78 const resolutionsEnabled: number[] = []
@@ -24,7 +90,7 @@ function computeResolutionsToTranscode (videoFileHeight: number) {
24 ] 90 ]
25 91
26 for (const resolution of resolutions) { 92 for (const resolution of resolutions) {
27 if (configResolutions[ resolution + 'p' ] === true && videoFileHeight > resolution) { 93 if (configResolutions[resolution + 'p'] === true && videoFileHeight > resolution) {
28 resolutionsEnabled.push(resolution) 94 resolutionsEnabled.push(resolution)
29 } 95 }
30 } 96 }
@@ -48,9 +114,9 @@ async function getVideoStreamCodec (path: string) {
48 const videoCodec = videoStream.codec_tag_string 114 const videoCodec = videoStream.codec_tag_string
49 115
50 const baseProfileMatrix = { 116 const baseProfileMatrix = {
51 'High': '6400', 117 High: '6400',
52 'Main': '4D40', 118 Main: '4D40',
53 'Baseline': '42E0' 119 Baseline: '42E0'
54 } 120 }
55 121
56 let baseProfile = baseProfileMatrix[videoStream.profile] 122 let baseProfile = baseProfileMatrix[videoStream.profile]
@@ -59,7 +125,8 @@ async function getVideoStreamCodec (path: string) {
59 baseProfile = baseProfileMatrix['High'] // Fallback 125 baseProfile = baseProfileMatrix['High'] // Fallback
60 } 126 }
61 127
62 const level = videoStream.level.toString(16) 128 let level = videoStream.level.toString(16)
129 if (level.length === 1) level = `0${level}`
63 130
64 return `${videoCodec}.${baseProfile}${level}` 131 return `${videoCodec}.${baseProfile}${level}`
65} 132}
@@ -91,7 +158,7 @@ async function getVideoFileFPS (path: string) {
91 if (videoStream === null) return 0 158 if (videoStream === null) return 0
92 159
93 for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) { 160 for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) {
94 const valuesText: string = videoStream[ key ] 161 const valuesText: string = videoStream[key]
95 if (!valuesText) continue 162 if (!valuesText) continue
96 163
97 const [ frames, seconds ] = valuesText.split('/') 164 const [ frames, seconds ] = valuesText.split('/')
@@ -104,24 +171,26 @@ async function getVideoFileFPS (path: string) {
104 return 0 171 return 0
105} 172}
106 173
107async function getVideoFileBitrate (path: string) { 174async function getMetadataFromFile <T> (path: string, cb = metadata => metadata) {
108 return new Promise<number>((res, rej) => { 175 return new Promise<T>((res, rej) => {
109 ffmpeg.ffprobe(path, (err, metadata) => { 176 ffmpeg.ffprobe(path, (err, metadata) => {
110 if (err) return rej(err) 177 if (err) return rej(err)
111 178
112 return res(metadata.format.bit_rate) 179 return res(cb(new VideoFileMetadata(metadata)))
113 }) 180 })
114 }) 181 })
115} 182}
116 183
184async function getVideoFileBitrate (path: string) {
185 return getMetadataFromFile<number>(path, metadata => metadata.format.bit_rate)
186}
187
117function getDurationFromVideoFile (path: string) { 188function getDurationFromVideoFile (path: string) {
118 return new Promise<number>((res, rej) => { 189 return getMetadataFromFile<number>(path, metadata => Math.floor(metadata.format.duration))
119 ffmpeg.ffprobe(path, (err, metadata) => { 190}
120 if (err) return rej(err)
121 191
122 return res(Math.floor(metadata.format.duration)) 192function getVideoStreamFromFile (path: string) {
123 }) 193 return getMetadataFromFile<any>(path, metadata => metadata.streams.find(s => s.codec_type === 'video') || null)
124 })
125} 194}
126 195
127async function generateImageFromVideoFile (fromPath: string, folder: string, imageName: string, size: { width: number, height: number }) { 196async function generateImageFromVideoFile (fromPath: string, folder: string, imageName: string, size: { width: number, height: number }) {
@@ -191,7 +260,8 @@ interface OnlyAudioTranscodeOptions extends BaseTranscodeOptions {
191 type: 'only-audio' 260 type: 'only-audio'
192} 261}
193 262
194type TranscodeOptions = HLSTranscodeOptions 263type TranscodeOptions =
264 HLSTranscodeOptions
195 | VideoTranscodeOptions 265 | VideoTranscodeOptions
196 | MergeAudioTranscodeOptions 266 | MergeAudioTranscodeOptions
197 | OnlyAudioTranscodeOptions 267 | OnlyAudioTranscodeOptions
@@ -204,13 +274,13 @@ function transcode (options: TranscodeOptions) {
204 .output(options.outputPath) 274 .output(options.outputPath)
205 275
206 if (options.type === 'quick-transcode') { 276 if (options.type === 'quick-transcode') {
207 command = await buildQuickTranscodeCommand(command) 277 command = buildQuickTranscodeCommand(command)
208 } else if (options.type === 'hls') { 278 } else if (options.type === 'hls') {
209 command = await buildHLSCommand(command, options) 279 command = await buildHLSCommand(command, options)
210 } else if (options.type === 'merge-audio') { 280 } else if (options.type === 'merge-audio') {
211 command = await buildAudioMergeCommand(command, options) 281 command = await buildAudioMergeCommand(command, options)
212 } else if (options.type === 'only-audio') { 282 } else if (options.type === 'only-audio') {
213 command = await buildOnlyAudioCommand(command, options) 283 command = buildOnlyAudioCommand(command, options)
214 } else { 284 } else {
215 command = await buildx264Command(command, options) 285 command = await buildx264Command(command, options)
216 } 286 }
@@ -247,22 +317,27 @@ async function canDoQuickTranscode (path: string): Promise<boolean> {
247 317
248 // check video params 318 // check video params
249 if (videoStream == null) return false 319 if (videoStream == null) return false
250 if (videoStream[ 'codec_name' ] !== 'h264') return false 320 if (videoStream['codec_name'] !== 'h264') return false
251 if (videoStream[ 'pix_fmt' ] !== 'yuv420p') return false 321 if (videoStream['pix_fmt'] !== 'yuv420p') return false
252 if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false 322 if (fps < VIDEO_TRANSCODING_FPS.MIN || fps > VIDEO_TRANSCODING_FPS.MAX) return false
253 if (bitRate > getMaxBitrate(resolution.videoFileResolution, fps, VIDEO_TRANSCODING_FPS)) return false 323 if (bitRate > getMaxBitrate(resolution.videoFileResolution, fps, VIDEO_TRANSCODING_FPS)) return false
254 324
255 // check audio params (if audio stream exists) 325 // check audio params (if audio stream exists)
256 if (parsedAudio.audioStream) { 326 if (parsedAudio.audioStream) {
257 if (parsedAudio.audioStream[ 'codec_name' ] !== 'aac') return false 327 if (parsedAudio.audioStream['codec_name'] !== 'aac') return false
258 328
259 const maxAudioBitrate = audio.bitrate[ 'aac' ](parsedAudio.audioStream[ 'bit_rate' ]) 329 const maxAudioBitrate = audio.bitrate['aac'](parsedAudio.audioStream['bit_rate'])
260 if (maxAudioBitrate !== -1 && parsedAudio.audioStream[ 'bit_rate' ] > maxAudioBitrate) return false 330 if (maxAudioBitrate !== -1 && parsedAudio.audioStream['bit_rate'] > maxAudioBitrate) return false
261 } 331 }
262 332
263 return true 333 return true
264} 334}
265 335
336function getClosestFramerateStandard (fps: number, type: 'HD_STANDARD' | 'STANDARD'): number {
337 return VIDEO_TRANSCODING_FPS[type].slice(0)
338 .sort((a, b) => fps % a - fps % b)[0]
339}
340
266// --------------------------------------------------------------------------- 341// ---------------------------------------------------------------------------
267 342
268export { 343export {
@@ -270,6 +345,7 @@ export {
270 getAudioStreamCodec, 345 getAudioStreamCodec,
271 getVideoStreamSize, 346 getVideoStreamSize,
272 getVideoFileResolution, 347 getVideoFileResolution,
348 getMetadataFromFile,
273 getDurationFromVideoFile, 349 getDurationFromVideoFile,
274 generateImageFromVideoFile, 350 generateImageFromVideoFile,
275 TranscodeOptions, 351 TranscodeOptions,
@@ -286,13 +362,14 @@ export {
286 362
287async function buildx264Command (command: ffmpeg.FfmpegCommand, options: TranscodeOptions) { 363async function buildx264Command (command: ffmpeg.FfmpegCommand, options: TranscodeOptions) {
288 let fps = await getVideoFileFPS(options.inputPath) 364 let fps = await getVideoFileFPS(options.inputPath)
289 // On small/medium resolutions, limit FPS
290 if ( 365 if (
366 // On small/medium resolutions, limit FPS
291 options.resolution !== undefined && 367 options.resolution !== undefined &&
292 options.resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN && 368 options.resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN &&
293 fps > VIDEO_TRANSCODING_FPS.AVERAGE 369 fps > VIDEO_TRANSCODING_FPS.AVERAGE
294 ) { 370 ) {
295 fps = VIDEO_TRANSCODING_FPS.AVERAGE 371 // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value
372 fps = getClosestFramerateStandard(fps, 'STANDARD')
296 } 373 }
297 374
298 command = await presetH264(command, options.inputPath, options.resolution, fps) 375 command = await presetH264(command, options.inputPath, options.resolution, fps)
@@ -305,7 +382,7 @@ async function buildx264Command (command: ffmpeg.FfmpegCommand, options: Transco
305 382
306 if (fps) { 383 if (fps) {
307 // Hard FPS limits 384 // Hard FPS limits
308 if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = VIDEO_TRANSCODING_FPS.MAX 385 if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard(fps, 'HD_STANDARD')
309 else if (fps < VIDEO_TRANSCODING_FPS.MIN) fps = VIDEO_TRANSCODING_FPS.MIN 386 else if (fps < VIDEO_TRANSCODING_FPS.MIN) fps = VIDEO_TRANSCODING_FPS.MIN
310 387
311 command = command.withFPS(fps) 388 command = command.withFPS(fps)
@@ -327,14 +404,14 @@ async function buildAudioMergeCommand (command: ffmpeg.FfmpegCommand, options: M
327 return command 404 return command
328} 405}
329 406
330async function buildOnlyAudioCommand (command: ffmpeg.FfmpegCommand, options: OnlyAudioTranscodeOptions) { 407function buildOnlyAudioCommand (command: ffmpeg.FfmpegCommand, options: OnlyAudioTranscodeOptions) {
331 command = await presetOnlyAudio(command) 408 command = presetOnlyAudio(command)
332 409
333 return command 410 return command
334} 411}
335 412
336async function buildQuickTranscodeCommand (command: ffmpeg.FfmpegCommand) { 413function buildQuickTranscodeCommand (command: ffmpeg.FfmpegCommand) {
337 command = await presetCopy(command) 414 command = presetCopy(command)
338 415
339 command = command.outputOption('-map_metadata -1') // strip all metadata 416 command = command.outputOption('-map_metadata -1') // strip all metadata
340 .outputOption('-movflags faststart') 417 .outputOption('-movflags faststart')
@@ -345,7 +422,8 @@ async function buildQuickTranscodeCommand (command: ffmpeg.FfmpegCommand) {
345async function buildHLSCommand (command: ffmpeg.FfmpegCommand, options: HLSTranscodeOptions) { 422async function buildHLSCommand (command: ffmpeg.FfmpegCommand, options: HLSTranscodeOptions) {
346 const videoPath = getHLSVideoPath(options) 423 const videoPath = getHLSVideoPath(options)
347 424
348 if (options.copyCodecs) command = await presetCopy(command) 425 if (options.copyCodecs) command = presetCopy(command)
426 else if (options.resolution === VideoResolution.H_NOVIDEO) command = presetOnlyAudio(command)
349 else command = await buildx264Command(command, options) 427 else command = await buildx264Command(command, options)
350 428
351 command = command.outputOption('-hls_time 4') 429 command = command.outputOption('-hls_time 4')
@@ -378,17 +456,6 @@ async function fixHLSPlaylistIfNeeded (options: TranscodeOptions) {
378 await writeFile(options.outputPath, newContent) 456 await writeFile(options.outputPath, newContent)
379} 457}
380 458
381function getVideoStreamFromFile (path: string) {
382 return new Promise<any>((res, rej) => {
383 ffmpeg.ffprobe(path, (err, metadata) => {
384 if (err) return rej(err)
385
386 const videoStream = metadata.streams.find(s => s.codec_type === 'video')
387 return res(videoStream || null)
388 })
389 })
390}
391
392/** 459/**
393 * A slightly customised version of the 'veryfast' x264 preset 460 * A slightly customised version of the 'veryfast' x264 preset
394 * 461 *
@@ -413,71 +480,6 @@ async function presetH264VeryFast (command: ffmpeg.FfmpegCommand, input: string,
413} 480}
414 481
415/** 482/**
416 * A toolbox to play with audio
417 */
418namespace audio {
419 export const get = (videoPath: string) => {
420 // without position, ffprobe considers the last input only
421 // we make it consider the first input only
422 // if you pass a file path to pos, then ffprobe acts on that file directly
423 return new Promise<{ absolutePath: string, audioStream?: any }>((res, rej) => {
424
425 function parseFfprobe (err: any, data: ffmpeg.FfprobeData) {
426 if (err) return rej(err)
427
428 if ('streams' in data) {
429 const audioStream = data.streams.find(stream => stream[ 'codec_type' ] === 'audio')
430 if (audioStream) {
431 return res({
432 absolutePath: data.format.filename,
433 audioStream
434 })
435 }
436 }
437
438 return res({ absolutePath: data.format.filename })
439 }
440
441 return ffmpeg.ffprobe(videoPath, parseFfprobe)
442 })
443 }
444
445 export namespace bitrate {
446 const baseKbitrate = 384
447
448 const toBits = (kbits: number) => kbits * 8000
449
450 export const aac = (bitrate: number): number => {
451 switch (true) {
452 case bitrate > toBits(baseKbitrate):
453 return baseKbitrate
454
455 default:
456 return -1 // we interpret it as a signal to copy the audio stream as is
457 }
458 }
459
460 export const mp3 = (bitrate: number): number => {
461 /*
462 a 192kbit/sec mp3 doesn't hold as much information as a 192kbit/sec aac.
463 That's why, when using aac, we can go to lower kbit/sec. The equivalences
464 made here are not made to be accurate, especially with good mp3 encoders.
465 */
466 switch (true) {
467 case bitrate <= toBits(192):
468 return 128
469
470 case bitrate <= toBits(384):
471 return 256
472
473 default:
474 return baseKbitrate
475 }
476 }
477 }
478}
479
480/**
481 * Standard profile, with variable bitrate audio and faststart. 483 * Standard profile, with variable bitrate audio and faststart.
482 * 484 *
483 * As for the audio, quality '5' is the highest and ensures 96-112kbps/channel 485 * As for the audio, quality '5' is the highest and ensures 96-112kbps/channel
@@ -507,10 +509,10 @@ async function presetH264 (command: ffmpeg.FfmpegCommand, input: string, resolut
507 // of course this is far from perfect, but it might save some space in the end 509 // of course this is far from perfect, but it might save some space in the end
508 localCommand = localCommand.audioCodec('aac') 510 localCommand = localCommand.audioCodec('aac')
509 511
510 const audioCodecName = parsedAudio.audioStream[ 'codec_name' ] 512 const audioCodecName = parsedAudio.audioStream['codec_name']
511 513
512 if (audio.bitrate[ audioCodecName ]) { 514 if (audio.bitrate[audioCodecName]) {
513 const bitrate = audio.bitrate[ audioCodecName ](parsedAudio.audioStream[ 'bit_rate' ]) 515 const bitrate = audio.bitrate[audioCodecName](parsedAudio.audioStream['bit_rate'])
514 if (bitrate !== undefined && bitrate !== -1) localCommand = localCommand.audioBitrate(bitrate) 516 if (bitrate !== undefined && bitrate !== -1) localCommand = localCommand.audioBitrate(bitrate)
515 } 517 }
516 } 518 }
@@ -531,14 +533,14 @@ async function presetH264 (command: ffmpeg.FfmpegCommand, input: string, resolut
531 return localCommand 533 return localCommand
532} 534}
533 535
534async function presetCopy (command: ffmpeg.FfmpegCommand): Promise<ffmpeg.FfmpegCommand> { 536function presetCopy (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
535 return command 537 return command
536 .format('mp4') 538 .format('mp4')
537 .videoCodec('copy') 539 .videoCodec('copy')
538 .audioCodec('copy') 540 .audioCodec('copy')
539} 541}
540 542
541async function presetOnlyAudio (command: ffmpeg.FfmpegCommand): Promise<ffmpeg.FfmpegCommand> { 543function presetOnlyAudio (command: ffmpeg.FfmpegCommand): ffmpeg.FfmpegCommand {
542 return command 544 return command
543 .format('mp4') 545 .format('mp4')
544 .audioCodec('copy') 546 .audioCodec('copy')
diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts
index 395417612..9553f70e8 100644
--- a/server/helpers/logger.ts
+++ b/server/helpers/logger.ts
@@ -5,7 +5,7 @@ import * as winston from 'winston'
5import { FileTransportOptions } from 'winston/lib/winston/transports' 5import { FileTransportOptions } from 'winston/lib/winston/transports'
6import { CONFIG } from '../initializers/config' 6import { CONFIG } from '../initializers/config'
7import { omit } from 'lodash' 7import { omit } from 'lodash'
8import { LOG_FILENAME } from '@server/initializers/constants' 8import { LOG_FILENAME } from '../initializers/constants'
9 9
10const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT 10const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
11 11
@@ -27,7 +27,7 @@ function getLoggerReplacer () {
27 if (value instanceof Error) { 27 if (value instanceof Error) {
28 const error = {} 28 const error = {}
29 29
30 Object.getOwnPropertyNames(value).forEach(key => error[ key ] = value[ key ]) 30 Object.getOwnPropertyNames(value).forEach(key => { error[key] = value[key] })
31 31
32 return error 32 return error
33 } 33 }
@@ -54,9 +54,11 @@ const jsonLoggerFormat = winston.format.printf(info => {
54const timestampFormatter = winston.format.timestamp({ 54const timestampFormatter = winston.format.timestamp({
55 format: 'YYYY-MM-DD HH:mm:ss.SSS' 55 format: 'YYYY-MM-DD HH:mm:ss.SSS'
56}) 56})
57const labelFormatter = winston.format.label({ 57const labelFormatter = (suffix?: string) => {
58 label 58 return winston.format.label({
59}) 59 label: suffix ? `${label} ${suffix}` : label
60 })
61}
60 62
61const fileLoggerOptions: FileTransportOptions = { 63const fileLoggerOptions: FileTransportOptions = {
62 filename: path.join(CONFIG.STORAGE.LOG_DIR, LOG_FILENAME), 64 filename: path.join(CONFIG.STORAGE.LOG_DIR, LOG_FILENAME),
@@ -72,25 +74,29 @@ if (CONFIG.LOG.ROTATION.ENABLED) {
72 fileLoggerOptions.maxFiles = CONFIG.LOG.ROTATION.MAX_FILES 74 fileLoggerOptions.maxFiles = CONFIG.LOG.ROTATION.MAX_FILES
73} 75}
74 76
75const logger = winston.createLogger({ 77const logger = buildLogger()
76 level: CONFIG.LOG.LEVEL, 78
77 format: winston.format.combine( 79function buildLogger (labelSuffix?: string) {
78 labelFormatter, 80 return winston.createLogger({
79 winston.format.splat() 81 level: CONFIG.LOG.LEVEL,
80 ), 82 format: winston.format.combine(
81 transports: [ 83 labelFormatter(labelSuffix),
82 new winston.transports.File(fileLoggerOptions), 84 winston.format.splat()
83 new winston.transports.Console({ 85 ),
84 handleExceptions: true, 86 transports: [
85 format: winston.format.combine( 87 new winston.transports.File(fileLoggerOptions),
86 timestampFormatter, 88 new winston.transports.Console({
87 winston.format.colorize(), 89 handleExceptions: true,
88 consoleLoggerFormat 90 format: winston.format.combine(
89 ) 91 timestampFormatter,
90 }) 92 winston.format.colorize(),
91 ], 93 consoleLoggerFormat
92 exitOnError: true 94 )
93}) 95 })
96 ],
97 exitOnError: true
98 })
99}
94 100
95function bunyanLogFactory (level: string) { 101function bunyanLogFactory (level: string) {
96 return function () { 102 return function () {
@@ -98,19 +104,20 @@ function bunyanLogFactory (level: string) {
98 let args: any[] = [] 104 let args: any[] = []
99 args.concat(arguments) 105 args.concat(arguments)
100 106
101 if (arguments[ 0 ] instanceof Error) { 107 if (arguments[0] instanceof Error) {
102 meta = arguments[ 0 ].toString() 108 meta = arguments[0].toString()
103 args = Array.prototype.slice.call(arguments, 1) 109 args = Array.prototype.slice.call(arguments, 1)
104 args.push(meta) 110 args.push(meta)
105 } else if (typeof (args[ 0 ]) !== 'string') { 111 } else if (typeof (args[0]) !== 'string') {
106 meta = arguments[ 0 ] 112 meta = arguments[0]
107 args = Array.prototype.slice.call(arguments, 1) 113 args = Array.prototype.slice.call(arguments, 1)
108 args.push(meta) 114 args.push(meta)
109 } 115 }
110 116
111 logger[ level ].apply(logger, args) 117 logger[level].apply(logger, args)
112 } 118 }
113} 119}
120
114const bunyanLogger = { 121const bunyanLogger = {
115 trace: bunyanLogFactory('debug'), 122 trace: bunyanLogFactory('debug'),
116 debug: bunyanLogFactory('debug'), 123 debug: bunyanLogFactory('debug'),
@@ -122,6 +129,7 @@ const bunyanLogger = {
122// --------------------------------------------------------------------------- 129// ---------------------------------------------------------------------------
123 130
124export { 131export {
132 buildLogger,
125 timestampFormatter, 133 timestampFormatter,
126 labelFormatter, 134 labelFormatter,
127 consoleLoggerFormat, 135 consoleLoggerFormat,
diff --git a/server/helpers/middlewares/video-abuses.ts b/server/helpers/middlewares/video-abuses.ts
index 8a1d3d618..97a5724b6 100644
--- a/server/helpers/middlewares/video-abuses.ts
+++ b/server/helpers/middlewares/video-abuses.ts
@@ -1,9 +1,17 @@
1import { Response } from 'express' 1import { Response } from 'express'
2import { VideoAbuseModel } from '../../models/video/video-abuse' 2import { VideoAbuseModel } from '../../models/video/video-abuse'
3import { fetchVideo } from '../video'
3 4
4async function doesVideoAbuseExist (abuseIdArg: number | string, videoId: number, res: Response) { 5async function doesVideoAbuseExist (abuseIdArg: number | string, videoUUID: string, res: Response) {
5 const abuseId = parseInt(abuseIdArg + '', 10) 6 const abuseId = parseInt(abuseIdArg + '', 10)
6 const videoAbuse = await VideoAbuseModel.loadByIdAndVideoId(abuseId, videoId) 7 let videoAbuse = await VideoAbuseModel.loadByIdAndVideoId(abuseId, null, videoUUID)
8
9 if (!videoAbuse) {
10 const userId = res.locals.oauth?.token.User.id
11 const video = await fetchVideo(videoUUID, 'all', userId)
12
13 if (video) videoAbuse = await VideoAbuseModel.loadByIdAndVideoId(abuseId, video.id)
14 }
7 15
8 if (videoAbuse === null) { 16 if (videoAbuse === null) {
9 res.status(404) 17 res.status(404)
diff --git a/server/helpers/middlewares/videos.ts b/server/helpers/middlewares/videos.ts
index 74f529804..a0bbcdb21 100644
--- a/server/helpers/middlewares/videos.ts
+++ b/server/helpers/middlewares/videos.ts
@@ -2,7 +2,17 @@ import { Response } from 'express'
2import { fetchVideo, VideoFetchType } from '../video' 2import { fetchVideo, VideoFetchType } from '../video'
3import { UserRight } from '../../../shared/models/users' 3import { UserRight } from '../../../shared/models/users'
4import { VideoChannelModel } from '../../models/video/video-channel' 4import { VideoChannelModel } from '../../models/video/video-channel'
5import { MUser, MUserAccountId, MVideoAccountLight, MVideoFullLight, MVideoThumbnail, MVideoWithRights } from '@server/typings/models' 5import {
6 MUser,
7 MUserAccountId,
8 MVideoAccountLight,
9 MVideoFullLight,
10 MVideoIdThumbnail,
11 MVideoImmutable,
12 MVideoThumbnail,
13 MVideoWithRights
14} from '@server/typings/models'
15import { VideoFileModel } from '@server/models/video/video-file'
6 16
7async function doesVideoExist (id: number | string, res: Response, fetchType: VideoFetchType = 'all') { 17async function doesVideoExist (id: number | string, res: Response, fetchType: VideoFetchType = 'all') {
8 const userId = res.locals.oauth ? res.locals.oauth.token.User.id : undefined 18 const userId = res.locals.oauth ? res.locals.oauth.token.User.id : undefined
@@ -22,8 +32,12 @@ async function doesVideoExist (id: number | string, res: Response, fetchType: Vi
22 res.locals.videoAll = video as MVideoFullLight 32 res.locals.videoAll = video as MVideoFullLight
23 break 33 break
24 34
35 case 'only-immutable-attributes':
36 res.locals.onlyImmutableVideo = video as MVideoImmutable
37 break
38
25 case 'id': 39 case 'id':
26 res.locals.videoId = video 40 res.locals.videoId = video as MVideoIdThumbnail
27 break 41 break
28 42
29 case 'only-video': 43 case 'only-video':
@@ -38,6 +52,18 @@ async function doesVideoExist (id: number | string, res: Response, fetchType: Vi
38 return true 52 return true
39} 53}
40 54
55async function doesVideoFileOfVideoExist (id: number, videoIdOrUUID: number | string, res: Response) {
56 if (!await VideoFileModel.doesVideoExistForVideoFile(id, videoIdOrUUID)) {
57 res.status(404)
58 .json({ error: 'VideoFile matching Video not found' })
59 .end()
60
61 return false
62 }
63
64 return true
65}
66
41async function doesVideoChannelOfAccountExist (channelId: number, user: MUserAccountId, res: Response) { 67async function doesVideoChannelOfAccountExist (channelId: number, user: MUserAccountId, res: Response) {
42 if (user.hasRight(UserRight.UPDATE_ANY_VIDEO) === true) { 68 if (user.hasRight(UserRight.UPDATE_ANY_VIDEO) === true) {
43 const videoChannel = await VideoChannelModel.loadAndPopulateAccount(channelId) 69 const videoChannel = await VideoChannelModel.loadAndPopulateAccount(channelId)
@@ -94,5 +120,6 @@ function checkUserCanManageVideo (user: MUser, video: MVideoAccountLight, right:
94export { 120export {
95 doesVideoChannelOfAccountExist, 121 doesVideoChannelOfAccountExist,
96 doesVideoExist, 122 doesVideoExist,
123 doesVideoFileOfVideoExist,
97 checkUserCanManageVideo 124 checkUserCanManageVideo
98} 125}
diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts
index 89c0ab151..394e97fd5 100644
--- a/server/helpers/peertube-crypto.ts
+++ b/server/helpers/peertube-crypto.ts
@@ -5,7 +5,6 @@ import { jsonld } from './custom-jsonld-signature'
5import { logger } from './logger' 5import { logger } from './logger'
6import { cloneDeep } from 'lodash' 6import { cloneDeep } from 'lodash'
7import { createSign, createVerify } from 'crypto' 7import { createSign, createVerify } from 'crypto'
8import { buildDigest } from '../lib/job-queue/handlers/utils/activitypub-http-utils'
9import * as bcrypt from 'bcrypt' 8import * as bcrypt from 'bcrypt'
10import { MActor } from '../typings/models' 9import { MActor } from '../typings/models'
11 10
@@ -104,12 +103,19 @@ async function signJsonLDObject (byActor: MActor, data: any) {
104 return Object.assign(data, { signature }) 103 return Object.assign(data, { signature })
105} 104}
106 105
106function buildDigest (body: any) {
107 const rawBody = typeof body === 'string' ? body : JSON.stringify(body)
108
109 return 'SHA-256=' + sha256(rawBody, 'base64')
110}
111
107// --------------------------------------------------------------------------- 112// ---------------------------------------------------------------------------
108 113
109export { 114export {
110 isHTTPSignatureDigestValid, 115 isHTTPSignatureDigestValid,
111 parseHTTPSignature, 116 parseHTTPSignature,
112 isHTTPSignatureVerified, 117 isHTTPSignatureVerified,
118 buildDigest,
113 isJsonLDSignatureVerified, 119 isJsonLDSignatureVerified,
114 comparePassword, 120 comparePassword,
115 createPrivateAndPublicKeys, 121 createPrivateAndPublicKeys,
diff --git a/server/helpers/regexp.ts b/server/helpers/regexp.ts
index 2336654b0..cfc2be488 100644
--- a/server/helpers/regexp.ts
+++ b/server/helpers/regexp.ts
@@ -1,8 +1,8 @@
1// Thanks to https://regex101.com 1// Thanks to https://regex101.com
2function regexpCapture (str: string, regex: RegExp, maxIterations = 100) { 2function regexpCapture (str: string, regex: RegExp, maxIterations = 100) {
3 const result: RegExpExecArray[] = []
3 let m: RegExpExecArray 4 let m: RegExpExecArray
4 let i = 0 5 let i = 0
5 let result: RegExpExecArray[] = []
6 6
7 // tslint:disable:no-conditional-assignment 7 // tslint:disable:no-conditional-assignment
8 while ((m = regex.exec(str)) !== null && i < maxIterations) { 8 while ((m = regex.exec(str)) !== null && i < maxIterations) {
diff --git a/server/helpers/register-ts-paths.ts b/server/helpers/register-ts-paths.ts
index e8db369e3..eec7fed3e 100644
--- a/server/helpers/register-ts-paths.ts
+++ b/server/helpers/register-ts-paths.ts
@@ -1,5 +1,5 @@
1import { resolve } from 'path' 1import { resolve } from 'path'
2const tsConfigPaths = require('tsconfig-paths') 2import tsConfigPaths = require('tsconfig-paths')
3 3
4const tsConfig = require('../../tsconfig.json') 4const tsConfig = require('../../tsconfig.json')
5 5
diff --git a/server/helpers/signup.ts b/server/helpers/signup.ts
index 7c73f7c5c..d34ff2db5 100644
--- a/server/helpers/signup.ts
+++ b/server/helpers/signup.ts
@@ -21,7 +21,7 @@ async function isSignupAllowed (): Promise<{ allowed: boolean, errorMessage?: st
21 21
22function isSignupAllowedForCurrentIP (ip: string) { 22function isSignupAllowedForCurrentIP (ip: string) {
23 const addr = ipaddr.parse(ip) 23 const addr = ipaddr.parse(ip)
24 let excludeList = [ 'blacklist' ] 24 const excludeList = [ 'blacklist' ]
25 let matched = '' 25 let matched = ''
26 26
27 // if there is a valid, non-empty whitelist, we exclude all unknown adresses too 27 // if there is a valid, non-empty whitelist, we exclude all unknown adresses too
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts
index 4c6f200f8..ad3b7949e 100644
--- a/server/helpers/utils.ts
+++ b/server/helpers/utils.ts
@@ -1,12 +1,11 @@
1import { ResultList } from '../../shared' 1import { ResultList } from '../../shared'
2import { ApplicationModel } from '../models/application/application' 2import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils'
3import { execPromise, execPromise2, pseudoRandomBytesPromise, sha256 } from './core-utils'
4import { logger } from './logger' 3import { logger } from './logger'
5import { join } from 'path' 4import { join } from 'path'
6import { Instance as ParseTorrent } from 'parse-torrent' 5import { Instance as ParseTorrent } from 'parse-torrent'
7import { remove } from 'fs-extra' 6import { remove } from 'fs-extra'
8import * as memoizee from 'memoizee'
9import { CONFIG } from '../initializers/config' 7import { CONFIG } from '../initializers/config'
8import { isVideoFileExtnameValid } from './custom-validators/videos'
10 9
11function deleteFileAsync (path: string) { 10function deleteFileAsync (path: string) {
12 remove(path) 11 remove(path)
@@ -14,7 +13,7 @@ function deleteFileAsync (path: string) {
14} 13}
15 14
16async function generateRandomString (size: number) { 15async function generateRandomString (size: number) {
17 const raw = await pseudoRandomBytesPromise(size) 16 const raw = await randomBytesPromise(size)
18 17
19 return raw.toString('hex') 18 return raw.toString('hex')
20} 19}
@@ -32,21 +31,18 @@ function getFormattedObjects<U, V, T extends FormattableToJSON<U, V>> (objects:
32 } as ResultList<V> 31 } as ResultList<V>
33} 32}
34 33
35const getServerActor = memoizee(async function () { 34function generateVideoImportTmpPath (target: string | ParseTorrent, extensionArg?: string) {
36 const application = await ApplicationModel.load() 35 const id = typeof target === 'string'
37 if (!application) throw Error('Could not load Application from database.') 36 ? target
37 : target.infoHash
38 38
39 const actor = application.Account.Actor 39 let extension = '.mp4'
40 actor.Account = application.Account 40 if (extensionArg && isVideoFileExtnameValid(extensionArg)) {
41 41 extension = extensionArg
42 return actor 42 }
43}, { promise: true })
44
45function generateVideoImportTmpPath (target: string | ParseTorrent) {
46 const id = typeof target === 'string' ? target : target.infoHash
47 43
48 const hash = sha256(id) 44 const hash = sha256(id)
49 return join(CONFIG.STORAGE.TMP_DIR, hash + '-import.mp4') 45 return join(CONFIG.STORAGE.TMP_DIR, `${hash}-import${extension}`)
50} 46}
51 47
52function getSecureTorrentName (originalName: string) { 48function getSecureTorrentName (originalName: string) {
@@ -97,7 +93,6 @@ export {
97 generateRandomString, 93 generateRandomString,
98 getFormattedObjects, 94 getFormattedObjects,
99 getSecureTorrentName, 95 getSecureTorrentName,
100 getServerActor,
101 getServerCommit, 96 getServerCommit,
102 generateVideoImportTmpPath, 97 generateVideoImportTmpPath,
103 getUUIDFromFilename 98 getUUIDFromFilename
diff --git a/server/helpers/video.ts b/server/helpers/video.ts
index 5b9c026b1..6f76cbdfc 100644
--- a/server/helpers/video.ts
+++ b/server/helpers/video.ts
@@ -1,17 +1,26 @@
1import { VideoModel } from '../models/video/video' 1import { VideoModel } from '../models/video/video'
2import * as Bluebird from 'bluebird' 2import * as Bluebird from 'bluebird'
3import { 3import {
4 isStreamingPlaylist,
5 MStreamingPlaylistVideo,
6 MVideo,
4 MVideoAccountLightBlacklistAllFiles, 7 MVideoAccountLightBlacklistAllFiles,
8 MVideoFile,
5 MVideoFullLight, 9 MVideoFullLight,
6 MVideoIdThumbnail, 10 MVideoIdThumbnail,
11 MVideoImmutable,
7 MVideoThumbnail, 12 MVideoThumbnail,
8 MVideoWithRights 13 MVideoWithRights
9} from '@server/typings/models' 14} from '@server/typings/models'
10import { Response } from 'express' 15import { Response } from 'express'
16import { DEFAULT_AUDIO_RESOLUTION } from '@server/initializers/constants'
17import { JobQueue } from '@server/lib/job-queue'
18import { VideoTranscodingPayload } from '@shared/models'
11 19
12type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none' 20type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none' | 'only-immutable-attributes'
13 21
14function fetchVideo (id: number | string, fetchType: 'all', userId?: number): Bluebird<MVideoFullLight> 22function fetchVideo (id: number | string, fetchType: 'all', userId?: number): Bluebird<MVideoFullLight>
23function fetchVideo (id: number | string, fetchType: 'only-immutable-attributes'): Bluebird<MVideoImmutable>
15function fetchVideo (id: number | string, fetchType: 'only-video', userId?: number): Bluebird<MVideoThumbnail> 24function fetchVideo (id: number | string, fetchType: 'only-video', userId?: number): Bluebird<MVideoThumbnail>
16function fetchVideo (id: number | string, fetchType: 'only-video-with-rights', userId?: number): Bluebird<MVideoWithRights> 25function fetchVideo (id: number | string, fetchType: 'only-video-with-rights', userId?: number): Bluebird<MVideoWithRights>
17function fetchVideo (id: number | string, fetchType: 'id' | 'none', userId?: number): Bluebird<MVideoIdThumbnail> 26function fetchVideo (id: number | string, fetchType: 'id' | 'none', userId?: number): Bluebird<MVideoIdThumbnail>
@@ -19,14 +28,16 @@ function fetchVideo (
19 id: number | string, 28 id: number | string,
20 fetchType: VideoFetchType, 29 fetchType: VideoFetchType,
21 userId?: number 30 userId?: number
22): Bluebird<MVideoFullLight | MVideoThumbnail | MVideoWithRights | MVideoIdThumbnail> 31): Bluebird<MVideoFullLight | MVideoThumbnail | MVideoWithRights | MVideoIdThumbnail | MVideoImmutable>
23function fetchVideo ( 32function fetchVideo (
24 id: number | string, 33 id: number | string,
25 fetchType: VideoFetchType, 34 fetchType: VideoFetchType,
26 userId?: number 35 userId?: number
27): Bluebird<MVideoFullLight | MVideoThumbnail | MVideoWithRights | MVideoIdThumbnail> { 36): Bluebird<MVideoFullLight | MVideoThumbnail | MVideoWithRights | MVideoIdThumbnail | MVideoImmutable> {
28 if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId) 37 if (fetchType === 'all') return VideoModel.loadAndPopulateAccountAndServerAndTags(id, undefined, userId)
29 38
39 if (fetchType === 'only-immutable-attributes') return VideoModel.loadImmutableAttributes(id)
40
30 if (fetchType === 'only-video-with-rights') return VideoModel.loadWithRights(id) 41 if (fetchType === 'only-video-with-rights') return VideoModel.loadWithRights(id)
31 42
32 if (fetchType === 'only-video') return VideoModel.load(id) 43 if (fetchType === 'only-video') return VideoModel.load(id)
@@ -34,14 +45,23 @@ function fetchVideo (
34 if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id) 45 if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id)
35} 46}
36 47
37type VideoFetchByUrlType = 'all' | 'only-video' 48type VideoFetchByUrlType = 'all' | 'only-video' | 'only-immutable-attributes'
38 49
39function fetchVideoByUrl (url: string, fetchType: 'all'): Bluebird<MVideoAccountLightBlacklistAllFiles> 50function fetchVideoByUrl (url: string, fetchType: 'all'): Bluebird<MVideoAccountLightBlacklistAllFiles>
51function fetchVideoByUrl (url: string, fetchType: 'only-immutable-attributes'): Bluebird<MVideoImmutable>
40function fetchVideoByUrl (url: string, fetchType: 'only-video'): Bluebird<MVideoThumbnail> 52function fetchVideoByUrl (url: string, fetchType: 'only-video'): Bluebird<MVideoThumbnail>
41function fetchVideoByUrl (url: string, fetchType: VideoFetchByUrlType): Bluebird<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail> 53function fetchVideoByUrl (
42function fetchVideoByUrl (url: string, fetchType: VideoFetchByUrlType): Bluebird<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail> { 54 url: string,
55 fetchType: VideoFetchByUrlType
56): Bluebird<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable>
57function fetchVideoByUrl (
58 url: string,
59 fetchType: VideoFetchByUrlType
60): Bluebird<MVideoAccountLightBlacklistAllFiles | MVideoThumbnail | MVideoImmutable> {
43 if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url) 61 if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url)
44 62
63 if (fetchType === 'only-immutable-attributes') return VideoModel.loadByUrlImmutableAttributes(url)
64
45 if (fetchType === 'only-video') return VideoModel.loadByUrl(url) 65 if (fetchType === 'only-video') return VideoModel.loadByUrl(url)
46} 66}
47 67
@@ -49,10 +69,39 @@ function getVideoWithAttributes (res: Response) {
49 return res.locals.videoAll || res.locals.onlyVideo || res.locals.onlyVideoWithRights 69 return res.locals.videoAll || res.locals.onlyVideo || res.locals.onlyVideoWithRights
50} 70}
51 71
72function addOptimizeOrMergeAudioJob (video: MVideo, videoFile: MVideoFile) {
73 let dataInput: VideoTranscodingPayload
74
75 if (videoFile.isAudio()) {
76 dataInput = {
77 type: 'merge-audio' as 'merge-audio',
78 resolution: DEFAULT_AUDIO_RESOLUTION,
79 videoUUID: video.uuid,
80 isNewVideo: true
81 }
82 } else {
83 dataInput = {
84 type: 'optimize' as 'optimize',
85 videoUUID: video.uuid,
86 isNewVideo: true
87 }
88 }
89
90 return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: dataInput })
91}
92
93function extractVideo (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
94 return isStreamingPlaylist(videoOrPlaylist)
95 ? videoOrPlaylist.Video
96 : videoOrPlaylist
97}
98
52export { 99export {
53 VideoFetchType, 100 VideoFetchType,
54 VideoFetchByUrlType, 101 VideoFetchByUrlType,
55 fetchVideo, 102 fetchVideo,
56 getVideoWithAttributes, 103 getVideoWithAttributes,
57 fetchVideoByUrl 104 fetchVideoByUrl,
105 addOptimizeOrMergeAudioJob,
106 extractVideo
58} 107}
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index 3a99518c6..7cd76d708 100644
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -9,12 +9,12 @@ import { promisify2 } from './core-utils'
9import { MVideo } from '@server/typings/models/video/video' 9import { MVideo } from '@server/typings/models/video/video'
10import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/typings/models/video/video-file' 10import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/typings/models/video/video-file'
11import { isStreamingPlaylist, MStreamingPlaylistVideo } from '@server/typings/models/video/video-streaming-playlist' 11import { isStreamingPlaylist, MStreamingPlaylistVideo } from '@server/typings/models/video/video-streaming-playlist'
12import { STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' 12import { WEBSERVER } from '@server/initializers/constants'
13import * as parseTorrent from 'parse-torrent' 13import * as parseTorrent from 'parse-torrent'
14import * as magnetUtil from 'magnet-uri' 14import * as magnetUtil from 'magnet-uri'
15import { isArray } from '@server/helpers/custom-validators/misc' 15import { isArray } from '@server/helpers/custom-validators/misc'
16import { extractVideo } from '@server/lib/videos' 16import { getTorrentFileName, getVideoFilePath } from '@server/lib/video-paths'
17import { getTorrentFileName, getVideoFilename, getVideoFilePath } from '@server/lib/video-paths' 17import { extractVideo } from '@server/helpers/video'
18 18
19const createTorrentPromise = promisify2<string, any, any>(createTorrent) 19const createTorrentPromise = promisify2<string, any, any>(createTorrent)
20 20
@@ -39,7 +39,7 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName
39 if (torrent.files.length !== 1) { 39 if (torrent.files.length !== 1) {
40 if (timer) clearTimeout(timer) 40 if (timer) clearTimeout(timer)
41 41
42 for (let file of torrent.files) { 42 for (const file of torrent.files) {
43 deleteDownloadedFile({ directoryPath, filepath: file.path }) 43 deleteDownloadedFile({ directoryPath, filepath: file.path })
44 } 44 }
45 45
@@ -47,15 +47,16 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName
47 .then(() => rej(new Error('Cannot import torrent ' + torrentId + ': there are multiple files in it'))) 47 .then(() => rej(new Error('Cannot import torrent ' + torrentId + ': there are multiple files in it')))
48 } 48 }
49 49
50 file = torrent.files[ 0 ] 50 file = torrent.files[0]
51 51
52 // FIXME: avoid creating another stream when https://github.com/webtorrent/webtorrent/issues/1517 is fixed 52 // FIXME: avoid creating another stream when https://github.com/webtorrent/webtorrent/issues/1517 is fixed
53 const writeStream = createWriteStream(path) 53 const writeStream = createWriteStream(path)
54 writeStream.on('finish', () => { 54 writeStream.on('finish', () => {
55 if (timer) clearTimeout(timer) 55 if (timer) clearTimeout(timer)
56 56
57 return safeWebtorrentDestroy(webtorrent, torrentId, { directoryPath, filepath: file.path }, target.torrentName) 57 safeWebtorrentDestroy(webtorrent, torrentId, { directoryPath, filepath: file.path }, target.torrentName)
58 .then(() => res(path)) 58 .then(() => res(path))
59 .catch(err => logger.error('Cannot destroy webtorrent.', { err }))
59 }) 60 })
60 61
61 file.createReadStream().pipe(writeStream) 62 file.createReadStream().pipe(writeStream)
@@ -63,9 +64,16 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName
63 64
64 torrent.on('error', err => rej(err)) 65 torrent.on('error', err => rej(err))
65 66
66 timer = setTimeout(async () => { 67 timer = setTimeout(() => {
67 return safeWebtorrentDestroy(webtorrent, torrentId, file ? { directoryPath, filepath: file.path } : undefined, target.torrentName) 68 const err = new Error('Webtorrent download timeout.')
68 .then(() => rej(new Error('Webtorrent download timeout.'))) 69
70 safeWebtorrentDestroy(webtorrent, torrentId, file ? { directoryPath, filepath: file.path } : undefined, target.torrentName)
71 .then(() => rej(err))
72 .catch(destroyErr => {
73 logger.error('Cannot destroy webtorrent.', { err: destroyErr })
74 rej(err)
75 })
76
69 }, timeout) 77 }, timeout)
70 }) 78 })
71} 79}
diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts
index 577a59dbf..f0944b94f 100644
--- a/server/helpers/youtube-dl.ts
+++ b/server/helpers/youtube-dl.ts
@@ -1,4 +1,4 @@
1import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers/constants' 1import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants'
2import { logger } from './logger' 2import { logger } from './logger'
3import { generateVideoImportTmpPath } from './utils' 3import { generateVideoImportTmpPath } from './utils'
4import { join } from 'path' 4import { join } from 'path'
@@ -12,40 +12,85 @@ export type YoutubeDLInfo = {
12 name?: string 12 name?: string
13 description?: string 13 description?: string
14 category?: number 14 category?: number
15 language?: string
15 licence?: number 16 licence?: number
16 nsfw?: boolean 17 nsfw?: boolean
17 tags?: string[] 18 tags?: string[]
18 thumbnailUrl?: string 19 thumbnailUrl?: string
20 fileExt?: string
19 originallyPublishedAt?: Date 21 originallyPublishedAt?: Date
20} 22}
21 23
24export type YoutubeDLSubs = {
25 language: string
26 filename: string
27 path: string
28}[]
29
22const processOptions = { 30const processOptions = {
23 maxBuffer: 1024 * 1024 * 10 // 10MB 31 maxBuffer: 1024 * 1024 * 10 // 10MB
24} 32}
25 33
26function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo> { 34function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo> {
27 return new Promise<YoutubeDLInfo>(async (res, rej) => { 35 return new Promise<YoutubeDLInfo>((res, rej) => {
28 let args = opts || [ '-j', '--flat-playlist' ] 36 let args = opts || [ '-j', '--flat-playlist' ]
29 args = wrapWithProxyOptions(args) 37 args = wrapWithProxyOptions(args)
30 38
31 const youtubeDL = await safeGetYoutubeDL() 39 safeGetYoutubeDL()
32 youtubeDL.getInfo(url, args, processOptions, (err, info) => { 40 .then(youtubeDL => {
33 if (err) return rej(err) 41 youtubeDL.getInfo(url, args, processOptions, (err, info) => {
34 if (info.is_live === true) return rej(new Error('Cannot download a live streaming.')) 42 if (err) return rej(err)
43 if (info.is_live === true) return rej(new Error('Cannot download a live streaming.'))
35 44
36 const obj = buildVideoInfo(normalizeObject(info)) 45 const obj = buildVideoInfo(normalizeObject(info))
37 if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video' 46 if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video'
38 47
39 return res(obj) 48 return res(obj)
40 }) 49 })
50 })
51 .catch(err => rej(err))
52 })
53}
54
55function getYoutubeDLSubs (url: string, opts?: object): Promise<YoutubeDLSubs> {
56 return new Promise<YoutubeDLSubs>((res, rej) => {
57 const cwd = CONFIG.STORAGE.TMP_DIR
58 const options = opts || { all: true, format: 'vtt', cwd }
59
60 safeGetYoutubeDL()
61 .then(youtubeDL => {
62 youtubeDL.getSubs(url, options, (err, files) => {
63 if (err) return rej(err)
64
65 logger.debug('Get subtitles from youtube dl.', { url, files })
66
67 const subtitles = files.reduce((acc, filename) => {
68 const matched = filename.match(/\.([a-z]{2})\.(vtt|ttml)/i)
69
70 if (matched[1]) {
71 return [
72 ...acc,
73 {
74 language: matched[1],
75 path: join(cwd, filename),
76 filename
77 }
78 ]
79 }
80 }, [])
81
82 return res(subtitles)
83 })
84 })
85 .catch(err => rej(err))
41 }) 86 })
42} 87}
43 88
44function downloadYoutubeDLVideo (url: string, timeout: number) { 89function downloadYoutubeDLVideo (url: string, extension: string, timeout: number) {
45 const path = generateVideoImportTmpPath(url) 90 const path = generateVideoImportTmpPath(url, extension)
46 let timer 91 let timer
47 92
48 logger.info('Importing youtubeDL video %s', url) 93 logger.info('Importing youtubeDL video %s to %s', url, path)
49 94
50 let options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ] 95 let options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]
51 options = wrapWithProxyOptions(options) 96 options = wrapWithProxyOptions(options)
@@ -54,26 +99,34 @@ function downloadYoutubeDLVideo (url: string, timeout: number) {
54 options = options.concat([ '--ffmpeg-location', process.env.FFMPEG_PATH ]) 99 options = options.concat([ '--ffmpeg-location', process.env.FFMPEG_PATH ])
55 } 100 }
56 101
57 return new Promise<string>(async (res, rej) => { 102 return new Promise<string>((res, rej) => {
58 const youtubeDL = await safeGetYoutubeDL() 103 safeGetYoutubeDL()
59 youtubeDL.exec(url, options, processOptions, err => { 104 .then(youtubeDL => {
60 clearTimeout(timer) 105 youtubeDL.exec(url, options, processOptions, err => {
106 clearTimeout(timer)
61 107
62 if (err) { 108 if (err) {
63 remove(path) 109 remove(path)
64 .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err })) 110 .catch(err => logger.error('Cannot delete path on YoutubeDL error.', { err }))
65 111
66 return rej(err) 112 return rej(err)
67 } 113 }
68 114
69 return res(path) 115 return res(path)
70 }) 116 })
71 117
72 timer = setTimeout(async () => { 118 timer = setTimeout(() => {
73 await remove(path) 119 const err = new Error('YoutubeDL download timeout.')
74 120
75 return rej(new Error('YoutubeDL download timeout.')) 121 remove(path)
76 }, timeout) 122 .finally(() => rej(err))
123 .catch(err => {
124 logger.error('Cannot remove %s in youtubeDL timeout.', path, { err })
125 return rej(err)
126 })
127 }, timeout)
128 })
129 .catch(err => rej(err))
77 }) 130 })
78} 131}
79 132
@@ -103,7 +156,7 @@ async function updateYoutubeDLBinary () {
103 156
104 const url = result.headers.location 157 const url = result.headers.location
105 const downloadFile = request.get(url) 158 const downloadFile = request.get(url)
106 const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[ 1 ] 159 const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[1]
107 160
108 downloadFile.on('response', result => { 161 downloadFile.on('response', result => {
109 if (result.statusCode !== 200) { 162 if (result.statusCode !== 200) {
@@ -173,6 +226,7 @@ function buildOriginallyPublishedAt (obj: any) {
173export { 226export {
174 updateYoutubeDLBinary, 227 updateYoutubeDLBinary,
175 downloadYoutubeDLVideo, 228 downloadYoutubeDLVideo,
229 getYoutubeDLSubs,
176 getYoutubeDLInfo, 230 getYoutubeDLInfo,
177 safeGetYoutubeDL, 231 safeGetYoutubeDL,
178 buildOriginallyPublishedAt 232 buildOriginallyPublishedAt
@@ -199,16 +253,18 @@ function normalizeObject (obj: any) {
199 return newObj 253 return newObj
200} 254}
201 255
202function buildVideoInfo (obj: any) { 256function buildVideoInfo (obj: any): YoutubeDLInfo {
203 return { 257 return {
204 name: titleTruncation(obj.title), 258 name: titleTruncation(obj.title),
205 description: descriptionTruncation(obj.description), 259 description: descriptionTruncation(obj.description),
206 category: getCategory(obj.categories), 260 category: getCategory(obj.categories),
207 licence: getLicence(obj.license), 261 licence: getLicence(obj.license),
262 language: getLanguage(obj.language),
208 nsfw: isNSFW(obj), 263 nsfw: isNSFW(obj),
209 tags: getTags(obj.tags), 264 tags: getTags(obj.tags),
210 thumbnailUrl: obj.thumbnail || undefined, 265 thumbnailUrl: obj.thumbnail || undefined,
211 originallyPublishedAt: buildOriginallyPublishedAt(obj) 266 originallyPublishedAt: buildOriginallyPublishedAt(obj),
267 fileExt: obj.ext
212 } 268 }
213} 269}
214 270
@@ -246,7 +302,12 @@ function getTags (tags: any) {
246function getLicence (licence: string) { 302function getLicence (licence: string) {
247 if (!licence) return undefined 303 if (!licence) return undefined
248 304
249 if (licence.indexOf('Creative Commons Attribution') !== -1) return 1 305 if (licence.includes('Creative Commons Attribution')) return 1
306
307 for (const key of Object.keys(VIDEO_LICENCES)) {
308 const peertubeLicence = VIDEO_LICENCES[key]
309 if (peertubeLicence.toLowerCase() === licence.toLowerCase()) return parseInt(key, 10)
310 }
250 311
251 return undefined 312 return undefined
252} 313}
@@ -267,6 +328,10 @@ function getCategory (categories: string[]) {
267 return undefined 328 return undefined
268} 329}
269 330
331function getLanguage (language: string) {
332 return VIDEO_LANGUAGES[language] ? language : undefined
333}
334
270function wrapWithProxyOptions (options: string[]) { 335function wrapWithProxyOptions (options: string[]) {
271 if (CONFIG.IMPORT.VIDEOS.HTTP.PROXY.ENABLED) { 336 if (CONFIG.IMPORT.VIDEOS.HTTP.PROXY.ENABLED) {
272 logger.debug('Using proxy for YoutubeDL') 337 logger.debug('Using proxy for YoutubeDL')