aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-08-31 17:18:13 +0200
committerChocobozzz <me@florianbigard.com>2018-08-31 17:22:01 +0200
commit9a629c6efbe39dfac290347670ca41b0d7100f41 (patch)
tree22c491983071aebe928fe73a2c51a8347fc78b8a
parent4ccb6c083025005a495e8ed40ea352c541f66951 (diff)
downloadPeerTube-9a629c6efbe39dfac290347670ca41b0d7100f41.tar.gz
PeerTube-9a629c6efbe39dfac290347670ca41b0d7100f41.tar.zst
PeerTube-9a629c6efbe39dfac290347670ca41b0d7100f41.zip
Trending by interval
-rw-r--r--client/src/app/shared/video/sort-field.type.ts1
-rw-r--r--client/src/app/videos/video-list/video-trending.component.ts2
-rw-r--r--config/default.yaml4
-rw-r--r--config/production.yaml.example4
-rw-r--r--server/initializers/checker.ts2
-rw-r--r--server/initializers/constants.ts10
-rw-r--r--server/models/utils.ts44
-rw-r--r--server/models/video/video.ts63
-rw-r--r--server/tests/api/videos/videos-overview.ts20
9 files changed, 116 insertions, 34 deletions
diff --git a/client/src/app/shared/video/sort-field.type.ts b/client/src/app/shared/video/sort-field.type.ts
index 2192745b9..d1088d244 100644
--- a/client/src/app/shared/video/sort-field.type.ts
+++ b/client/src/app/shared/video/sort-field.type.ts
@@ -4,3 +4,4 @@ export type VideoSortField = 'name' | '-name'
4 | 'createdAt' | '-createdAt' 4 | 'createdAt' | '-createdAt'
5 | 'views' | '-views' 5 | 'views' | '-views'
6 | 'likes' | '-likes' 6 | 'likes' | '-likes'
7 | 'trending' | '-trending'
diff --git a/client/src/app/videos/video-list/video-trending.component.ts b/client/src/app/videos/video-list/video-trending.component.ts
index 68bb70265..8f3d3842b 100644
--- a/client/src/app/videos/video-list/video-trending.component.ts
+++ b/client/src/app/videos/video-list/video-trending.component.ts
@@ -18,7 +18,7 @@ import { ScreenService } from '@app/shared/misc/screen.service'
18export class VideoTrendingComponent extends AbstractVideoList implements OnInit, OnDestroy { 18export class VideoTrendingComponent extends AbstractVideoList implements OnInit, OnDestroy {
19 titlePage: string 19 titlePage: string
20 currentRoute = '/videos/trending' 20 currentRoute = '/videos/trending'
21 defaultSort: VideoSortField = '-views' 21 defaultSort: VideoSortField = '-trending'
22 22
23 constructor ( 23 constructor (
24 protected router: Router, 24 protected router: Router,
diff --git a/config/default.yaml b/config/default.yaml
index ef63fbd28..254fa0c99 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -62,6 +62,10 @@ search:
62 users: true 62 users: true
63 anonymous: false 63 anonymous: false
64 64
65trending:
66 videos:
67 interval_days: 7 # Compute trending videos for the last x days
68
65cache: 69cache:
66 previews: 70 previews:
67 size: 500 # Max number of previews you want to cache 71 size: 500 # Max number of previews you want to cache
diff --git a/config/production.yaml.example b/config/production.yaml.example
index f7b153698..e33427fae 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -63,6 +63,10 @@ search:
63 users: true 63 users: true
64 anonymous: false 64 anonymous: false
65 65
66trending:
67 videos:
68 interval_days: 7 # Compute trending videos for the last x days
69
66############################################################################### 70###############################################################################
67# 71#
68# From this point, all the following keys can be overridden by the web interface 72# From this point, all the following keys can be overridden by the web interface
diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts
index ee02ecf48..b126bf67e 100644
--- a/server/initializers/checker.ts
+++ b/server/initializers/checker.ts
@@ -52,7 +52,7 @@ function checkMissedConfig () {
52 'signup.enabled', 'signup.limit', 'signup.requires_email_verification', 52 'signup.enabled', 'signup.limit', 'signup.requires_email_verification',
53 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', 53 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist',
54 'transcoding.enabled', 'transcoding.threads', 54 'transcoding.enabled', 'transcoding.threads',
55 'import.videos.http.enabled', 55 'import.videos.http.enabled', 'import.videos.torrent.enabled',
56 'instance.name', 'instance.short_description', 'instance.description', 'instance.terms', 'instance.default_client_route', 56 'instance.name', 'instance.short_description', 'instance.description', 'instance.terms', 'instance.default_client_route',
57 'instance.default_nsfw_policy', 'instance.robots', 57 'instance.default_nsfw_policy', 'instance.robots',
58 'services.twitter.username', 'services.twitter.whitelisted' 58 'services.twitter.username', 'services.twitter.whitelisted'
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 6d0503f48..efe27a241 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -37,14 +37,15 @@ const SORTABLE_COLUMNS = {
37 JOBS: [ 'createdAt' ], 37 JOBS: [ 'createdAt' ],
38 VIDEO_ABUSES: [ 'id', 'createdAt', 'state' ], 38 VIDEO_ABUSES: [ 'id', 'createdAt', 'state' ],
39 VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ], 39 VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ],
40 VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'views', 'likes' ],
41 VIDEO_IMPORTS: [ 'createdAt' ], 40 VIDEO_IMPORTS: [ 'createdAt' ],
42 VIDEO_COMMENT_THREADS: [ 'createdAt' ], 41 VIDEO_COMMENT_THREADS: [ 'createdAt' ],
43 BLACKLISTS: [ 'id', 'name', 'duration', 'views', 'likes', 'dislikes', 'uuid', 'createdAt' ], 42 BLACKLISTS: [ 'id', 'name', 'duration', 'views', 'likes', 'dislikes', 'uuid', 'createdAt' ],
44 FOLLOWERS: [ 'createdAt' ], 43 FOLLOWERS: [ 'createdAt' ],
45 FOLLOWING: [ 'createdAt' ], 44 FOLLOWING: [ 'createdAt' ],
46 45
47 VIDEOS_SEARCH: [ 'match', 'name', 'duration', 'createdAt', 'publishedAt', 'views', 'likes' ], 46 VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'views', 'likes', 'trending' ],
47
48 VIDEOS_SEARCH: [ 'name', 'duration', 'createdAt', 'publishedAt', 'views', 'likes', 'match' ],
48 VIDEO_CHANNELS_SEARCH: [ 'match', 'displayName', 'createdAt' ] 49 VIDEO_CHANNELS_SEARCH: [ 'match', 'displayName', 'createdAt' ]
49} 50}
50 51
@@ -201,6 +202,11 @@ const CONFIG = {
201 ANONYMOUS: config.get<boolean>('search.remote_uri.anonymous') 202 ANONYMOUS: config.get<boolean>('search.remote_uri.anonymous')
202 } 203 }
203 }, 204 },
205 TRENDING: {
206 VIDEOS: {
207 INTERVAL_DAYS: config.get<number>('trending.videos.interval_days')
208 }
209 },
204 ADMIN: { 210 ADMIN: {
205 get EMAIL () { return config.get<string>('admin.email') } 211 get EMAIL () { return config.get<string>('admin.email') }
206 }, 212 },
diff --git a/server/models/utils.ts b/server/models/utils.ts
index eb6653f3d..edb8e1161 100644
--- a/server/models/utils.ts
+++ b/server/models/utils.ts
@@ -1,23 +1,31 @@
1// Translate for example "-name" to [ [ 'name', 'DESC' ], [ 'id', 'ASC' ] ]
2import { Sequelize } from 'sequelize-typescript' 1import { Sequelize } from 'sequelize-typescript'
3 2
4type SortType = { sortModel: any, sortValue: string } 3type SortType = { sortModel: any, sortValue: string }
5 4
5// Translate for example "-name" to [ [ 'name', 'DESC' ], [ 'id', 'ASC' ] ]
6function getSort (value: string, lastSort: string[] = [ 'id', 'ASC' ]) { 6function getSort (value: string, lastSort: string[] = [ 'id', 'ASC' ]) {
7 let field: any 7 const { direction, field } = buildDirectionAndField(value)
8 let direction: 'ASC' | 'DESC'
9 8
10 if (value.substring(0, 1) === '-') { 9 return [ [ field, direction ], lastSort ]
11 direction = 'DESC' 10}
12 field = value.substring(1) 11
13 } else { 12function getVideoSort (value: string, lastSort: string[] = [ 'id', 'ASC' ]) {
14 direction = 'ASC' 13 let { direction, field } = buildDirectionAndField(value)
15 field = value
16 }
17 14
18 // Alias 15 // Alias
19 if (field.toLowerCase() === 'match') field = Sequelize.col('similarity') 16 if (field.toLowerCase() === 'match') field = Sequelize.col('similarity')
20 17
18 // Sort by aggregation
19 if (field.toLowerCase() === 'trending') {
20 return [
21 [ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoViews.views')), '0'), direction ],
22
23 [ Sequelize.col('VideoModel.views'), direction ],
24
25 lastSort
26 ]
27 }
28
21 return [ [ field, direction ], lastSort ] 29 return [ [ field, direction ], lastSort ]
22} 30}
23 31
@@ -58,6 +66,7 @@ function createSimilarityAttribute (col: string, value: string) {
58export { 66export {
59 SortType, 67 SortType,
60 getSort, 68 getSort,
69 getVideoSort,
61 getSortOnModel, 70 getSortOnModel,
62 createSimilarityAttribute, 71 createSimilarityAttribute,
63 throwIfNotValid, 72 throwIfNotValid,
@@ -73,3 +82,18 @@ function searchTrigramNormalizeValue (value: string) {
73function searchTrigramNormalizeCol (col: string) { 82function searchTrigramNormalizeCol (col: string) {
74 return Sequelize.fn('lower', Sequelize.fn('immutable_unaccent', Sequelize.col(col))) 83 return Sequelize.fn('lower', Sequelize.fn('immutable_unaccent', Sequelize.col(col)))
75} 84}
85
86function buildDirectionAndField (value: string) {
87 let field: any
88 let direction: 'ASC' | 'DESC'
89
90 if (value.substring(0, 1) === '-') {
91 direction = 'DESC'
92 field = value.substring(1)
93 } else {
94 direction = 'ASC'
95 field = value
96 }
97
98 return { direction, field }
99}
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 67b123d77..6fb5ececa 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -17,6 +17,7 @@ import {
17 HasMany, 17 HasMany,
18 HasOne, 18 HasOne,
19 IFindOptions, 19 IFindOptions,
20 IIncludeOptions,
20 Is, 21 Is,
21 IsInt, 22 IsInt,
22 IsUUID, 23 IsUUID,
@@ -24,8 +25,7 @@ import {
24 Model, 25 Model,
25 Scopes, 26 Scopes,
26 Table, 27 Table,
27 UpdatedAt, 28 UpdatedAt
28 IIncludeOptions
29} from 'sequelize-typescript' 29} from 'sequelize-typescript'
30import { VideoPrivacy, VideoResolution, VideoState } from '../../../shared' 30import { VideoPrivacy, VideoResolution, VideoState } from '../../../shared'
31import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' 31import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
@@ -77,7 +77,7 @@ import { AccountVideoRateModel } from '../account/account-video-rate'
77import { ActorModel } from '../activitypub/actor' 77import { ActorModel } from '../activitypub/actor'
78import { AvatarModel } from '../avatar/avatar' 78import { AvatarModel } from '../avatar/avatar'
79import { ServerModel } from '../server/server' 79import { ServerModel } from '../server/server'
80import { buildTrigramSearchIndex, createSimilarityAttribute, getSort, throwIfNotValid } from '../utils' 80import { buildTrigramSearchIndex, createSimilarityAttribute, getVideoSort, throwIfNotValid } from '../utils'
81import { TagModel } from './tag' 81import { TagModel } from './tag'
82import { VideoAbuseModel } from './video-abuse' 82import { VideoAbuseModel } from './video-abuse'
83import { VideoChannelModel } from './video-channel' 83import { VideoChannelModel } from './video-channel'
@@ -89,7 +89,7 @@ import { ScheduleVideoUpdateModel } from './schedule-video-update'
89import { VideoCaptionModel } from './video-caption' 89import { VideoCaptionModel } from './video-caption'
90import { VideoBlacklistModel } from './video-blacklist' 90import { VideoBlacklistModel } from './video-blacklist'
91import { copy, remove, rename, stat, writeFile } from 'fs-extra' 91import { copy, remove, rename, stat, writeFile } from 'fs-extra'
92import { immutableAssign } from '../../tests/utils' 92import { VideoViewModel } from './video-views'
93 93
94// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation 94// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
95const indexes: Sequelize.DefineIndexesOptions[] = [ 95const indexes: Sequelize.DefineIndexesOptions[] = [
@@ -146,6 +146,7 @@ type AvailableForListIDsOptions = {
146 withFiles?: boolean 146 withFiles?: boolean
147 accountId?: number 147 accountId?: number
148 videoChannelId?: number 148 videoChannelId?: number
149 trendingDays?: number
149} 150}
150 151
151@Scopes({ 152@Scopes({
@@ -384,6 +385,21 @@ type AvailableForListIDsOptions = {
384 } 385 }
385 } 386 }
386 387
388 if (options.trendingDays) {
389 query.include.push({
390 attributes: [],
391 model: VideoViewModel,
392 required: false,
393 where: {
394 startDate: {
395 [ Sequelize.Op.gte ]: new Date(new Date().getTime() - (24 * 3600 * 1000) * options.trendingDays)
396 }
397 }
398 })
399
400 query.subQuery = false
401 }
402
387 return query 403 return query
388 }, 404 },
389 [ScopeNames.WITH_ACCOUNT_DETAILS]: { 405 [ScopeNames.WITH_ACCOUNT_DETAILS]: {
@@ -649,6 +665,16 @@ export class VideoModel extends Model<VideoModel> {
649 }) 665 })
650 VideoComments: VideoCommentModel[] 666 VideoComments: VideoCommentModel[]
651 667
668 @HasMany(() => VideoViewModel, {
669 foreignKey: {
670 name: 'videoId',
671 allowNull: false
672 },
673 onDelete: 'cascade',
674 hooks: true
675 })
676 VideoViews: VideoViewModel[]
677
652 @HasOne(() => ScheduleVideoUpdateModel, { 678 @HasOne(() => ScheduleVideoUpdateModel, {
653 foreignKey: { 679 foreignKey: {
654 name: 'videoId', 680 name: 'videoId',
@@ -754,7 +780,7 @@ export class VideoModel extends Model<VideoModel> {
754 distinct: true, 780 distinct: true,
755 offset: start, 781 offset: start,
756 limit: count, 782 limit: count,
757 order: getSort('createdAt', [ 'Tags', 'name', 'ASC' ]), 783 order: getVideoSort('createdAt', [ 'Tags', 'name', 'ASC' ]),
758 where: { 784 where: {
759 id: { 785 id: {
760 [Sequelize.Op.in]: Sequelize.literal('(' + rawQuery + ')') 786 [Sequelize.Op.in]: Sequelize.literal('(' + rawQuery + ')')
@@ -845,7 +871,7 @@ export class VideoModel extends Model<VideoModel> {
845 const query: IFindOptions<VideoModel> = { 871 const query: IFindOptions<VideoModel> = {
846 offset: start, 872 offset: start,
847 limit: count, 873 limit: count,
848 order: getSort(sort), 874 order: getVideoSort(sort),
849 include: [ 875 include: [
850 { 876 {
851 model: VideoChannelModel, 877 model: VideoChannelModel,
@@ -902,11 +928,19 @@ export class VideoModel extends Model<VideoModel> {
902 accountId?: number, 928 accountId?: number,
903 videoChannelId?: number, 929 videoChannelId?: number,
904 actorId?: number 930 actorId?: number
931 trendingDays?: number
905 }) { 932 }) {
906 const query = { 933 const query: IFindOptions<VideoModel> = {
907 offset: options.start, 934 offset: options.start,
908 limit: options.count, 935 limit: options.count,
909 order: getSort(options.sort) 936 order: getVideoSort(options.sort)
937 }
938
939 let trendingDays: number
940 if (options.sort.endsWith('trending')) {
941 trendingDays = CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS
942
943 query.group = 'VideoModel.id'
910 } 944 }
911 945
912 // actorId === null has a meaning, so just check undefined 946 // actorId === null has a meaning, so just check undefined
@@ -924,7 +958,8 @@ export class VideoModel extends Model<VideoModel> {
924 withFiles: options.withFiles, 958 withFiles: options.withFiles,
925 accountId: options.accountId, 959 accountId: options.accountId,
926 videoChannelId: options.videoChannelId, 960 videoChannelId: options.videoChannelId,
927 includeLocalVideos: options.includeLocalVideos 961 includeLocalVideos: options.includeLocalVideos,
962 trendingDays
928 } 963 }
929 964
930 return VideoModel.getAvailableForApi(query, queryOptions) 965 return VideoModel.getAvailableForApi(query, queryOptions)
@@ -1006,7 +1041,7 @@ export class VideoModel extends Model<VideoModel> {
1006 }, 1041 },
1007 offset: options.start, 1042 offset: options.start,
1008 limit: options.count, 1043 limit: options.count,
1009 order: getSort(options.sort), 1044 order: getVideoSort(options.sort),
1010 where: { 1045 where: {
1011 [ Sequelize.Op.and ]: whereAnd 1046 [ Sequelize.Op.and ]: whereAnd
1012 } 1047 }
@@ -1177,8 +1212,12 @@ export class VideoModel extends Model<VideoModel> {
1177 const secondQuery = { 1212 const secondQuery = {
1178 offset: 0, 1213 offset: 0,
1179 limit: query.limit, 1214 limit: query.limit,
1180 order: query.order, 1215 attributes: query.attributes,
1181 attributes: query.attributes 1216 order: [ // Keep original order
1217 Sequelize.literal(
1218 ids.map(id => `"VideoModel".id = ${id} DESC`).join(', ')
1219 )
1220 ]
1182 } 1221 }
1183 const rows = await VideoModel.scope(apiScope).findAll(secondQuery) 1222 const rows = await VideoModel.scope(apiScope).findAll(secondQuery)
1184 1223
diff --git a/server/tests/api/videos/videos-overview.ts b/server/tests/api/videos/videos-overview.ts
index 1514d1bda..7d1f29c92 100644
--- a/server/tests/api/videos/videos-overview.ts
+++ b/server/tests/api/videos/videos-overview.ts
@@ -30,8 +30,10 @@ describe('Test a videos overview', function () {
30 expect(overview.channels).to.have.lengthOf(0) 30 expect(overview.channels).to.have.lengthOf(0)
31 }) 31 })
32 32
33 it('Should upload 3 videos in a specific category, tag and channel but not include them in overview', async function () { 33 it('Should upload 5 videos in a specific category, tag and channel but not include them in overview', async function () {
34 for (let i = 0; i < 3; i++) { 34 this.timeout(15000)
35
36 for (let i = 0; i < 5; i++) {
35 await uploadVideo(server.url, server.accessToken, { 37 await uploadVideo(server.url, server.accessToken, {
36 name: 'video ' + i, 38 name: 'video ' + i,
37 category: 3, 39 category: 3,
@@ -49,7 +51,7 @@ describe('Test a videos overview', function () {
49 51
50 it('Should upload another video and include all videos in the overview', async function () { 52 it('Should upload another video and include all videos in the overview', async function () {
51 await uploadVideo(server.url, server.accessToken, { 53 await uploadVideo(server.url, server.accessToken, {
52 name: 'video 3', 54 name: 'video 5',
53 category: 3, 55 category: 3,
54 tags: [ 'coucou1', 'coucou2' ] 56 tags: [ 'coucou1', 'coucou2' ]
55 }) 57 })
@@ -70,11 +72,13 @@ describe('Test a videos overview', function () {
70 for (const attr of [ 'tags', 'categories', 'channels' ]) { 72 for (const attr of [ 'tags', 'categories', 'channels' ]) {
71 const obj = overview[attr][0] 73 const obj = overview[attr][0]
72 74
73 expect(obj.videos).to.have.lengthOf(4) 75 expect(obj.videos).to.have.lengthOf(6)
74 expect(obj.videos[0].name).to.equal('video 3') 76 expect(obj.videos[0].name).to.equal('video 5')
75 expect(obj.videos[1].name).to.equal('video 2') 77 expect(obj.videos[1].name).to.equal('video 4')
76 expect(obj.videos[2].name).to.equal('video 1') 78 expect(obj.videos[2].name).to.equal('video 3')
77 expect(obj.videos[3].name).to.equal('video 0') 79 expect(obj.videos[3].name).to.equal('video 2')
80 expect(obj.videos[4].name).to.equal('video 1')
81 expect(obj.videos[5].name).to.equal('video 0')
78 } 82 }
79 83
80 expect(overview.tags.find(t => t.tag === 'coucou1')).to.not.be.undefined 84 expect(overview.tags.find(t => t.tag === 'coucou1')).to.not.be.undefined