aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/overviews.ts70
-rw-r--r--server/initializers/constants.ts9
-rw-r--r--server/middlewares/validators/videos/videos.ts19
-rw-r--r--server/tests/api/check-params/index.ts1
-rw-r--r--server/tests/api/check-params/videos-overviews.ts33
-rw-r--r--server/tests/api/videos/video-nsfw.ts38
-rw-r--r--server/tests/api/videos/videos-overview.ts85
7 files changed, 185 insertions, 70 deletions
diff --git a/server/controllers/api/overviews.ts b/server/controllers/api/overviews.ts
index 75f3baedb..fb31932aa 100644
--- a/server/controllers/api/overviews.ts
+++ b/server/controllers/api/overviews.ts
@@ -1,17 +1,18 @@
1import * as express from 'express' 1import * as express from 'express'
2import { buildNSFWFilter } from '../../helpers/express-utils' 2import { buildNSFWFilter } from '../../helpers/express-utils'
3import { VideoModel } from '../../models/video/video' 3import { VideoModel } from '../../models/video/video'
4import { asyncMiddleware } from '../../middlewares' 4import { asyncMiddleware, optionalAuthenticate, videosOverviewValidator } from '../../middlewares'
5import { TagModel } from '../../models/video/tag' 5import { TagModel } from '../../models/video/tag'
6import { VideosOverview } from '../../../shared/models/overviews' 6import { CategoryOverview, ChannelOverview, TagOverview, VideosOverview } from '../../../shared/models/overviews'
7import { MEMOIZE_TTL, OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers/constants' 7import { MEMOIZE_TTL, OVERVIEWS } from '../../initializers/constants'
8import { cacheRoute } from '../../middlewares/cache'
9import * as memoizee from 'memoizee' 8import * as memoizee from 'memoizee'
9import { logger } from '@server/helpers/logger'
10 10
11const overviewsRouter = express.Router() 11const overviewsRouter = express.Router()
12 12
13overviewsRouter.get('/videos', 13overviewsRouter.get('/videos',
14 asyncMiddleware(cacheRoute()(ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS)), 14 videosOverviewValidator,
15 optionalAuthenticate,
15 asyncMiddleware(getVideosOverview) 16 asyncMiddleware(getVideosOverview)
16) 17)
17 18
@@ -28,17 +29,28 @@ const buildSamples = memoizee(async function () {
28 TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT) 29 TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT)
29 ]) 30 ])
30 31
31 return { categories, channels, tags } 32 const result = { categories, channels, tags }
33
34 logger.debug('Building samples for overview endpoint.', { result })
35
36 return result
32}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE }) 37}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE })
33 38
34// This endpoint could be quite long, but we cache it 39// This endpoint could be quite long, but we cache it
35async function getVideosOverview (req: express.Request, res: express.Response) { 40async function getVideosOverview (req: express.Request, res: express.Response) {
36 const attributes = await buildSamples() 41 const attributes = await buildSamples()
37 42
38 const [ categories, channels, tags ] = await Promise.all([ 43 const page = req.query.page || 1
39 Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))), 44 const index = page - 1
40 Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))), 45
41 Promise.all(attributes.tags.map(t => getVideosByTag(t, res))) 46 const categories: CategoryOverview[] = []
47 const channels: ChannelOverview[] = []
48 const tags: TagOverview[] = []
49
50 await Promise.all([
51 getVideosByCategory(attributes.categories, index, res, categories),
52 getVideosByChannel(attributes.channels, index, res, channels),
53 getVideosByTag(attributes.tags, index, res, tags)
42 ]) 54 ])
43 55
44 const result: VideosOverview = { 56 const result: VideosOverview = {
@@ -47,45 +59,49 @@ async function getVideosOverview (req: express.Request, res: express.Response) {
47 tags 59 tags
48 } 60 }
49 61
50 // Cleanup our object
51 for (const key of Object.keys(result)) {
52 result[key] = result[key].filter(v => v !== undefined)
53 }
54
55 return res.json(result) 62 return res.json(result)
56} 63}
57 64
58async function getVideosByTag (tag: string, res: express.Response) { 65async function getVideosByTag (tagsSample: string[], index: number, res: express.Response, acc: TagOverview[]) {
66 if (tagsSample.length <= index) return
67
68 const tag = tagsSample[index]
59 const videos = await getVideos(res, { tagsOneOf: [ tag ] }) 69 const videos = await getVideos(res, { tagsOneOf: [ tag ] })
60 70
61 if (videos.length === 0) return undefined 71 if (videos.length === 0) return
62 72
63 return { 73 acc.push({
64 tag, 74 tag,
65 videos 75 videos
66 } 76 })
67} 77}
68 78
69async function getVideosByCategory (category: number, res: express.Response) { 79async function getVideosByCategory (categoriesSample: number[], index: number, res: express.Response, acc: CategoryOverview[]) {
80 if (categoriesSample.length <= index) return
81
82 const category = categoriesSample[index]
70 const videos = await getVideos(res, { categoryOneOf: [ category ] }) 83 const videos = await getVideos(res, { categoryOneOf: [ category ] })
71 84
72 if (videos.length === 0) return undefined 85 if (videos.length === 0) return
73 86
74 return { 87 acc.push({
75 category: videos[0].category, 88 category: videos[0].category,
76 videos 89 videos
77 } 90 })
78} 91}
79 92
80async function getVideosByChannel (channelId: number, res: express.Response) { 93async function getVideosByChannel (channelsSample: number[], index: number, res: express.Response, acc: ChannelOverview[]) {
94 if (channelsSample.length <= index) return
95
96 const channelId = channelsSample[index]
81 const videos = await getVideos(res, { videoChannelId: channelId }) 97 const videos = await getVideos(res, { videoChannelId: channelId })
82 98
83 if (videos.length === 0) return undefined 99 if (videos.length === 0) return
84 100
85 return { 101 acc.push({
86 channel: videos[0].channel, 102 channel: videos[0].channel,
87 videos 103 videos
88 } 104 })
89} 105}
90 106
91async function getVideos ( 107async function getVideos (
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 8b040aa2c..13448ffed 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -90,9 +90,6 @@ const ROUTE_CACHE_LIFETIME = {
90 SECURITYTXT: '2 hours', 90 SECURITYTXT: '2 hours',
91 NODEINFO: '10 minutes', 91 NODEINFO: '10 minutes',
92 DNT_POLICY: '1 week', 92 DNT_POLICY: '1 week',
93 OVERVIEWS: {
94 VIDEOS: '1 hour'
95 },
96 ACTIVITY_PUB: { 93 ACTIVITY_PUB: {
97 VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example 94 VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example
98 }, 95 },
@@ -446,7 +443,7 @@ MIMETYPES.IMAGE.EXT_MIMETYPE = invert(MIMETYPES.IMAGE.MIMETYPE_EXT)
446const OVERVIEWS = { 443const OVERVIEWS = {
447 VIDEOS: { 444 VIDEOS: {
448 SAMPLE_THRESHOLD: 6, 445 SAMPLE_THRESHOLD: 6,
449 SAMPLES_COUNT: 2 446 SAMPLES_COUNT: 20
450 } 447 }
451} 448}
452 449
@@ -687,8 +684,8 @@ if (isTestInstance() === true) {
687 JOB_ATTEMPTS['email'] = 1 684 JOB_ATTEMPTS['email'] = 1
688 685
689 FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000 686 FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000
690 MEMOIZE_TTL.OVERVIEWS_SAMPLE = 1 687 MEMOIZE_TTL.OVERVIEWS_SAMPLE = 3000
691 ROUTE_CACHE_LIFETIME.OVERVIEWS.VIDEOS = '0ms' 688 OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD = 2
692} 689}
693 690
694updateWebserverUrls() 691updateWebserverUrls()
diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts
index 96e0d6600..3a7869354 100644
--- a/server/middlewares/validators/videos/videos.ts
+++ b/server/middlewares/validators/videos/videos.ts
@@ -29,7 +29,7 @@ import {
29} from '../../../helpers/custom-validators/videos' 29} from '../../../helpers/custom-validators/videos'
30import { getDurationFromVideoFile } from '../../../helpers/ffmpeg-utils' 30import { getDurationFromVideoFile } from '../../../helpers/ffmpeg-utils'
31import { logger } from '../../../helpers/logger' 31import { logger } from '../../../helpers/logger'
32import { CONSTRAINTS_FIELDS } from '../../../initializers/constants' 32import { CONSTRAINTS_FIELDS, OVERVIEWS } from '../../../initializers/constants'
33import { authenticatePromiseIfNeeded } from '../../oauth' 33import { authenticatePromiseIfNeeded } from '../../oauth'
34import { areValidationErrors } from '../utils' 34import { areValidationErrors } from '../utils'
35import { cleanUpReqFiles } from '../../../helpers/express-utils' 35import { cleanUpReqFiles } from '../../../helpers/express-utils'
@@ -301,6 +301,19 @@ const videosAcceptChangeOwnershipValidator = [
301 } 301 }
302] 302]
303 303
304const videosOverviewValidator = [
305 query('page')
306 .optional()
307 .isInt({ min: 1, max: OVERVIEWS.VIDEOS.SAMPLES_COUNT })
308 .withMessage('Should have a valid pagination'),
309
310 (req: express.Request, res: express.Response, next: express.NextFunction) => {
311 if (areValidationErrors(req, res)) return
312
313 return next()
314 }
315]
316
304function getCommonVideoEditAttributes () { 317function getCommonVideoEditAttributes () {
305 return [ 318 return [
306 body('thumbnailfile') 319 body('thumbnailfile')
@@ -442,7 +455,9 @@ export {
442 455
443 getCommonVideoEditAttributes, 456 getCommonVideoEditAttributes,
444 457
445 commonVideosFiltersValidator 458 commonVideosFiltersValidator,
459
460 videosOverviewValidator
446} 461}
447 462
448// --------------------------------------------------------------------------- 463// ---------------------------------------------------------------------------
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts
index 924c0df76..ef152f55c 100644
--- a/server/tests/api/check-params/index.ts
+++ b/server/tests/api/check-params/index.ts
@@ -23,3 +23,4 @@ import './video-playlists'
23import './videos' 23import './videos'
24import './videos-filter' 24import './videos-filter'
25import './videos-history' 25import './videos-history'
26import './videos-overviews'
diff --git a/server/tests/api/check-params/videos-overviews.ts b/server/tests/api/check-params/videos-overviews.ts
new file mode 100644
index 000000000..69d7fc471
--- /dev/null
+++ b/server/tests/api/check-params/videos-overviews.ts
@@ -0,0 +1,33 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import 'mocha'
4import { cleanupTests, flushAndRunServer, ServerInfo } from '../../../../shared/extra-utils'
5import { getVideosOverview } from '@shared/extra-utils/overviews/overviews'
6
7describe('Test videos overview', function () {
8 let server: ServerInfo
9
10 // ---------------------------------------------------------------
11
12 before(async function () {
13 this.timeout(30000)
14
15 server = await flushAndRunServer(1)
16 })
17
18 describe('When getting videos overview', function () {
19
20 it('Should fail with a bad pagination', async function () {
21 await getVideosOverview(server.url, 0, 400)
22 await getVideosOverview(server.url, 100, 400)
23 })
24
25 it('Should succeed with a good pagination', async function () {
26 await getVideosOverview(server.url, 1)
27 })
28 })
29
30 after(async function () {
31 await cleanupTests([ server ])
32 })
33})
diff --git a/server/tests/api/videos/video-nsfw.ts b/server/tests/api/videos/video-nsfw.ts
index 7eba8d7d9..b16b484b9 100644
--- a/server/tests/api/videos/video-nsfw.ts
+++ b/server/tests/api/videos/video-nsfw.ts
@@ -19,12 +19,20 @@ import {
19 updateCustomConfig, 19 updateCustomConfig,
20 updateMyUser 20 updateMyUser
21} from '../../../../shared/extra-utils' 21} from '../../../../shared/extra-utils'
22import { ServerConfig } from '../../../../shared/models' 22import { ServerConfig, VideosOverview } from '../../../../shared/models'
23import { CustomConfig } from '../../../../shared/models/server/custom-config.model' 23import { CustomConfig } from '../../../../shared/models/server/custom-config.model'
24import { User } from '../../../../shared/models/users' 24import { User } from '../../../../shared/models/users'
25import { getVideosOverview, getVideosOverviewWithToken } from '@shared/extra-utils/overviews/overviews'
25 26
26const expect = chai.expect 27const expect = chai.expect
27 28
29function createOverviewRes (res: any) {
30 const overview = res.body as VideosOverview
31
32 const videos = overview.categories[0].videos
33 return { body: { data: videos, total: videos.length } }
34}
35
28describe('Test video NSFW policy', function () { 36describe('Test video NSFW policy', function () {
29 let server: ServerInfo 37 let server: ServerInfo
30 let userAccessToken: string 38 let userAccessToken: string
@@ -36,22 +44,38 @@ describe('Test video NSFW policy', function () {
36 const user: User = res.body 44 const user: User = res.body
37 const videoChannelName = user.videoChannels[0].name 45 const videoChannelName = user.videoChannels[0].name
38 const accountName = user.account.name + '@' + user.account.host 46 const accountName = user.account.name + '@' + user.account.host
47 const hasQuery = Object.keys(query).length !== 0
48 let promises: Promise<any>[]
39 49
40 if (token) { 50 if (token) {
41 return Promise.all([ 51 promises = [
42 getVideosListWithToken(server.url, token, query), 52 getVideosListWithToken(server.url, token, query),
43 searchVideoWithToken(server.url, 'n', token, query), 53 searchVideoWithToken(server.url, 'n', token, query),
44 getAccountVideos(server.url, token, accountName, 0, 5, undefined, query), 54 getAccountVideos(server.url, token, accountName, 0, 5, undefined, query),
45 getVideoChannelVideos(server.url, token, videoChannelName, 0, 5, undefined, query) 55 getVideoChannelVideos(server.url, token, videoChannelName, 0, 5, undefined, query)
46 ]) 56 ]
57
58 // Overviews do not support video filters
59 if (!hasQuery) {
60 promises.push(getVideosOverviewWithToken(server.url, 1, token).then(res => createOverviewRes(res)))
61 }
62
63 return Promise.all(promises)
47 } 64 }
48 65
49 return Promise.all([ 66 promises = [
50 getVideosList(server.url), 67 getVideosList(server.url),
51 searchVideo(server.url, 'n'), 68 searchVideo(server.url, 'n'),
52 getAccountVideos(server.url, undefined, accountName, 0, 5), 69 getAccountVideos(server.url, undefined, accountName, 0, 5),
53 getVideoChannelVideos(server.url, undefined, videoChannelName, 0, 5) 70 getVideoChannelVideos(server.url, undefined, videoChannelName, 0, 5)
54 ]) 71 ]
72
73 // Overviews do not support video filters
74 if (!hasQuery) {
75 promises.push(getVideosOverview(server.url, 1).then(res => createOverviewRes(res)))
76 }
77
78 return Promise.all(promises)
55 }) 79 })
56 } 80 }
57 81
@@ -63,12 +87,12 @@ describe('Test video NSFW policy', function () {
63 await setAccessTokensToServers([ server ]) 87 await setAccessTokensToServers([ server ])
64 88
65 { 89 {
66 const attributes = { name: 'nsfw', nsfw: true } 90 const attributes = { name: 'nsfw', nsfw: true, category: 1 }
67 await uploadVideo(server.url, server.accessToken, attributes) 91 await uploadVideo(server.url, server.accessToken, attributes)
68 } 92 }
69 93
70 { 94 {
71 const attributes = { name: 'normal', nsfw: false } 95 const attributes = { name: 'normal', nsfw: false, category: 1 }
72 await uploadVideo(server.url, server.accessToken, attributes) 96 await uploadVideo(server.url, server.accessToken, attributes)
73 } 97 }
74 98
diff --git a/server/tests/api/videos/videos-overview.ts b/server/tests/api/videos/videos-overview.ts
index ca08ab5b1..d38bcb6eb 100644
--- a/server/tests/api/videos/videos-overview.ts
+++ b/server/tests/api/videos/videos-overview.ts
@@ -2,7 +2,7 @@
2 2
3import * as chai from 'chai' 3import * as chai from 'chai'
4import 'mocha' 4import 'mocha'
5import { cleanupTests, flushAndRunServer, ServerInfo, setAccessTokensToServers, uploadVideo } from '../../../../shared/extra-utils' 5import { cleanupTests, flushAndRunServer, ServerInfo, setAccessTokensToServers, uploadVideo, wait } from '../../../../shared/extra-utils'
6import { getVideosOverview } from '../../../../shared/extra-utils/overviews/overviews' 6import { getVideosOverview } from '../../../../shared/extra-utils/overviews/overviews'
7import { VideosOverview } from '../../../../shared/models/overviews' 7import { VideosOverview } from '../../../../shared/models/overviews'
8 8
@@ -20,7 +20,7 @@ describe('Test a videos overview', function () {
20 }) 20 })
21 21
22 it('Should send empty overview', async function () { 22 it('Should send empty overview', async function () {
23 const res = await getVideosOverview(server.url) 23 const res = await getVideosOverview(server.url, 1)
24 24
25 const overview: VideosOverview = res.body 25 const overview: VideosOverview = res.body
26 expect(overview.tags).to.have.lengthOf(0) 26 expect(overview.tags).to.have.lengthOf(0)
@@ -31,15 +31,15 @@ describe('Test a videos overview', function () {
31 it('Should upload 5 videos in a specific category, tag and channel but not include them in overview', async function () { 31 it('Should upload 5 videos in a specific category, tag and channel but not include them in overview', async function () {
32 this.timeout(15000) 32 this.timeout(15000)
33 33
34 for (let i = 0; i < 5; i++) { 34 await wait(3000)
35 await uploadVideo(server.url, server.accessToken, { 35
36 name: 'video ' + i, 36 await uploadVideo(server.url, server.accessToken, {
37 category: 3, 37 name: 'video 0',
38 tags: [ 'coucou1', 'coucou2' ] 38 category: 3,
39 }) 39 tags: [ 'coucou1', 'coucou2' ]
40 } 40 })
41 41
42 const res = await getVideosOverview(server.url) 42 const res = await getVideosOverview(server.url, 1)
43 43
44 const overview: VideosOverview = res.body 44 const overview: VideosOverview = res.body
45 expect(overview.tags).to.have.lengthOf(0) 45 expect(overview.tags).to.have.lengthOf(0)
@@ -48,27 +48,55 @@ describe('Test a videos overview', function () {
48 }) 48 })
49 49
50 it('Should upload another video and include all videos in the overview', async function () { 50 it('Should upload another video and include all videos in the overview', async function () {
51 await uploadVideo(server.url, server.accessToken, { 51 this.timeout(15000)
52 name: 'video 5',
53 category: 3,
54 tags: [ 'coucou1', 'coucou2' ]
55 })
56 52
57 const res = await getVideosOverview(server.url) 53 for (let i = 1; i < 6; i++) {
54 await uploadVideo(server.url, server.accessToken, {
55 name: 'video ' + i,
56 category: 3,
57 tags: [ 'coucou1', 'coucou2' ]
58 })
59 }
58 60
59 const overview: VideosOverview = res.body 61 await wait(3000)
60 expect(overview.tags).to.have.lengthOf(2) 62
61 expect(overview.categories).to.have.lengthOf(1) 63 {
62 expect(overview.channels).to.have.lengthOf(1) 64 const res = await getVideosOverview(server.url, 1)
65
66 const overview: VideosOverview = res.body
67 expect(overview.tags).to.have.lengthOf(1)
68 expect(overview.categories).to.have.lengthOf(1)
69 expect(overview.channels).to.have.lengthOf(1)
70 }
71
72 {
73 const res = await getVideosOverview(server.url, 2)
74
75 const overview: VideosOverview = res.body
76 expect(overview.tags).to.have.lengthOf(1)
77 expect(overview.categories).to.have.lengthOf(0)
78 expect(overview.channels).to.have.lengthOf(0)
79 }
63 }) 80 })
64 81
65 it('Should have the correct overview', async function () { 82 it('Should have the correct overview', async function () {
66 const res = await getVideosOverview(server.url) 83 const res1 = await getVideosOverview(server.url, 1)
84 const res2 = await getVideosOverview(server.url, 2)
67 85
68 const overview: VideosOverview = res.body 86 const overview1: VideosOverview = res1.body
87 const overview2: VideosOverview = res2.body
88
89 const tmp = [
90 overview1.tags,
91 overview1.categories,
92 overview1.channels,
93 overview2.tags
94 ]
95
96 for (const arr of tmp) {
97 expect(arr).to.have.lengthOf(1)
69 98
70 for (const attr of [ 'tags', 'categories', 'channels' ]) { 99 const obj = arr[0]
71 const obj = overview[attr][0]
72 100
73 expect(obj.videos).to.have.lengthOf(6) 101 expect(obj.videos).to.have.lengthOf(6)
74 expect(obj.videos[0].name).to.equal('video 5') 102 expect(obj.videos[0].name).to.equal('video 5')
@@ -79,12 +107,13 @@ describe('Test a videos overview', function () {
79 expect(obj.videos[5].name).to.equal('video 0') 107 expect(obj.videos[5].name).to.equal('video 0')
80 } 108 }
81 109
82 expect(overview.tags.find(t => t.tag === 'coucou1')).to.not.be.undefined 110 const tags = [ overview1.tags[0].tag, overview2.tags[0].tag ]
83 expect(overview.tags.find(t => t.tag === 'coucou2')).to.not.be.undefined 111 expect(tags.find(t => t === 'coucou1')).to.not.be.undefined
112 expect(tags.find(t => t === 'coucou2')).to.not.be.undefined
84 113
85 expect(overview.categories[0].category.id).to.equal(3) 114 expect(overview1.categories[0].category.id).to.equal(3)
86 115
87 expect(overview.channels[0].channel.name).to.equal('root_channel') 116 expect(overview1.channels[0].channel.name).to.equal('root_channel')
88 }) 117 })
89 118
90 after(async function () { 119 after(async function () {