diff options
author | Chocobozzz <me@florianbigard.com> | 2022-03-24 13:36:47 +0100 |
---|---|---|
committer | Chocobozzz <chocobozzz@cpy.re> | 2022-04-15 09:49:35 +0200 |
commit | b211106695bb82f6c32e53306081b5262c3d109d (patch) | |
tree | fa187de1c33b0956665f5362e29af6b0f6d8bb57 /server | |
parent | 69d48ee30c9d47cddf0c3c047dc99a99dcb6e894 (diff) | |
download | PeerTube-b211106695bb82f6c32e53306081b5262c3d109d.tar.gz PeerTube-b211106695bb82f6c32e53306081b5262c3d109d.tar.zst PeerTube-b211106695bb82f6c32e53306081b5262c3d109d.zip |
Support video views/viewers stats in server
* Add "currentTime" and "event" body params to view endpoint
* Merge watching and view endpoints
* Introduce WatchAction AP activity
* Add tables to store viewer information of local videos
* Add endpoints to fetch video views/viewers stats of local videos
* Refactor views/viewers handlers
* Support "views" and "viewers" counters for both VOD and live videos
Diffstat (limited to 'server')
79 files changed, 2598 insertions, 593 deletions
diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts index d0f761009..8e064fb5b 100644 --- a/server/controllers/activitypub/client.ts +++ b/server/controllers/activitypub/client.ts | |||
@@ -27,7 +27,7 @@ import { | |||
27 | videosShareValidator | 27 | videosShareValidator |
28 | } from '../../middlewares' | 28 | } from '../../middlewares' |
29 | import { cacheRoute } from '../../middlewares/cache/cache' | 29 | import { cacheRoute } from '../../middlewares/cache/cache' |
30 | import { getAccountVideoRateValidatorFactory, videoCommentGetValidator } from '../../middlewares/validators' | 30 | import { getAccountVideoRateValidatorFactory, getVideoLocalViewerValidator, videoCommentGetValidator } from '../../middlewares/validators' |
31 | import { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy' | 31 | import { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy' |
32 | import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '../../middlewares/validators/videos/video-playlists' | 32 | import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '../../middlewares/validators/videos/video-playlists' |
33 | import { AccountModel } from '../../models/account/account' | 33 | import { AccountModel } from '../../models/account/account' |
@@ -175,6 +175,12 @@ activityPubClientRouter.get('/video-playlists/:playlistId/videos/:playlistElemen | |||
175 | videoPlaylistElementController | 175 | videoPlaylistElementController |
176 | ) | 176 | ) |
177 | 177 | ||
178 | activityPubClientRouter.get('/videos/local-viewer/:localViewerId', | ||
179 | executeIfActivityPub, | ||
180 | asyncMiddleware(getVideoLocalViewerValidator), | ||
181 | getVideoLocalViewerController | ||
182 | ) | ||
183 | |||
178 | // --------------------------------------------------------------------------- | 184 | // --------------------------------------------------------------------------- |
179 | 185 | ||
180 | export { | 186 | export { |
@@ -399,6 +405,12 @@ function videoPlaylistElementController (req: express.Request, res: express.Resp | |||
399 | return activityPubResponse(activityPubContextify(json, 'Playlist'), res) | 405 | return activityPubResponse(activityPubContextify(json, 'Playlist'), res) |
400 | } | 406 | } |
401 | 407 | ||
408 | function getVideoLocalViewerController (req: express.Request, res: express.Response) { | ||
409 | const localViewer = res.locals.localViewerFull | ||
410 | |||
411 | return activityPubResponse(activityPubContextify(localViewer.toActivityPubObject(), 'WatchAction'), res) | ||
412 | } | ||
413 | |||
402 | // --------------------------------------------------------------------------- | 414 | // --------------------------------------------------------------------------- |
403 | 415 | ||
404 | function actorFollowing (req: express.Request, actor: MActorId) { | 416 | function actorFollowing (req: express.Request, actor: MActorId) { |
diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts index 093e6a03c..6b6ff027c 100644 --- a/server/controllers/api/server/debug.ts +++ b/server/controllers/api/server/debug.ts | |||
@@ -1,6 +1,8 @@ | |||
1 | import express from 'express' | 1 | import express from 'express' |
2 | import { InboxManager } from '@server/lib/activitypub/inbox-manager' | 2 | import { InboxManager } from '@server/lib/activitypub/inbox-manager' |
3 | import { RemoveDanglingResumableUploadsScheduler } from '@server/lib/schedulers/remove-dangling-resumable-uploads-scheduler' | 3 | import { RemoveDanglingResumableUploadsScheduler } from '@server/lib/schedulers/remove-dangling-resumable-uploads-scheduler' |
4 | import { VideoViewsBufferScheduler } from '@server/lib/schedulers/video-views-buffer-scheduler' | ||
5 | import { VideoViewsManager } from '@server/lib/views/video-views-manager' | ||
4 | import { Debug, SendDebugCommand } from '@shared/models' | 6 | import { Debug, SendDebugCommand } from '@shared/models' |
5 | import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' | 7 | import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' |
6 | import { UserRight } from '../../../../shared/models/users' | 8 | import { UserRight } from '../../../../shared/models/users' |
@@ -38,9 +40,13 @@ function getDebug (req: express.Request, res: express.Response) { | |||
38 | async function runCommand (req: express.Request, res: express.Response) { | 40 | async function runCommand (req: express.Request, res: express.Response) { |
39 | const body: SendDebugCommand = req.body | 41 | const body: SendDebugCommand = req.body |
40 | 42 | ||
41 | if (body.command === 'remove-dandling-resumable-uploads') { | 43 | const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = { |
42 | await RemoveDanglingResumableUploadsScheduler.Instance.execute() | 44 | 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(), |
45 | 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(), | ||
46 | 'process-video-viewers': () => VideoViewsManager.Instance.processViewers() | ||
43 | } | 47 | } |
44 | 48 | ||
49 | await processors[body.command]() | ||
50 | |||
45 | return res.status(HttpStatusCode.NO_CONTENT_204).end() | 51 | return res.status(HttpStatusCode.NO_CONTENT_204).end() |
46 | } | 52 | } |
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index c7617093c..be233722c 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts | |||
@@ -1,7 +1,6 @@ | |||
1 | import express from 'express' | 1 | import express from 'express' |
2 | import { pickCommonVideoQuery } from '@server/helpers/query' | 2 | import { pickCommonVideoQuery } from '@server/helpers/query' |
3 | import { doJSONRequest } from '@server/helpers/requests' | 3 | import { doJSONRequest } from '@server/helpers/requests' |
4 | import { VideoViews } from '@server/lib/video-views' | ||
5 | import { openapiOperationDoc } from '@server/middlewares/doc' | 4 | import { openapiOperationDoc } from '@server/middlewares/doc' |
6 | import { getServerActor } from '@server/models/application/application' | 5 | import { getServerActor } from '@server/models/application/application' |
7 | import { guessAdditionalAttributesFromQuery } from '@server/models/video/formatter/video-format-utils' | 6 | import { guessAdditionalAttributesFromQuery } from '@server/models/video/formatter/video-format-utils' |
@@ -13,7 +12,6 @@ import { logger } from '../../../helpers/logger' | |||
13 | import { getFormattedObjects } from '../../../helpers/utils' | 12 | import { getFormattedObjects } from '../../../helpers/utils' |
14 | import { REMOTE_SCHEME, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../initializers/constants' | 13 | import { REMOTE_SCHEME, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../initializers/constants' |
15 | import { sequelizeTypescript } from '../../../initializers/database' | 14 | import { sequelizeTypescript } from '../../../initializers/database' |
16 | import { sendView } from '../../../lib/activitypub/send/send-view' | ||
17 | import { JobQueue } from '../../../lib/job-queue' | 15 | import { JobQueue } from '../../../lib/job-queue' |
18 | import { Hooks } from '../../../lib/plugins/hooks' | 16 | import { Hooks } from '../../../lib/plugins/hooks' |
19 | import { | 17 | import { |
@@ -35,28 +33,30 @@ import { VideoModel } from '../../../models/video/video' | |||
35 | import { blacklistRouter } from './blacklist' | 33 | import { blacklistRouter } from './blacklist' |
36 | import { videoCaptionsRouter } from './captions' | 34 | import { videoCaptionsRouter } from './captions' |
37 | import { videoCommentRouter } from './comment' | 35 | import { videoCommentRouter } from './comment' |
38 | import { studioRouter } from './studio' | ||
39 | import { filesRouter } from './files' | 36 | import { filesRouter } from './files' |
40 | import { videoImportsRouter } from './import' | 37 | import { videoImportsRouter } from './import' |
41 | import { liveRouter } from './live' | 38 | import { liveRouter } from './live' |
42 | import { ownershipVideoRouter } from './ownership' | 39 | import { ownershipVideoRouter } from './ownership' |
43 | import { rateVideoRouter } from './rate' | 40 | import { rateVideoRouter } from './rate' |
41 | import { statsRouter } from './stats' | ||
42 | import { studioRouter } from './studio' | ||
44 | import { transcodingRouter } from './transcoding' | 43 | import { transcodingRouter } from './transcoding' |
45 | import { updateRouter } from './update' | 44 | import { updateRouter } from './update' |
46 | import { uploadRouter } from './upload' | 45 | import { uploadRouter } from './upload' |
47 | import { watchingRouter } from './watching' | 46 | import { viewRouter } from './view' |
48 | 47 | ||
49 | const auditLogger = auditLoggerFactory('videos') | 48 | const auditLogger = auditLoggerFactory('videos') |
50 | const videosRouter = express.Router() | 49 | const videosRouter = express.Router() |
51 | 50 | ||
52 | videosRouter.use('/', blacklistRouter) | 51 | videosRouter.use('/', blacklistRouter) |
52 | videosRouter.use('/', statsRouter) | ||
53 | videosRouter.use('/', rateVideoRouter) | 53 | videosRouter.use('/', rateVideoRouter) |
54 | videosRouter.use('/', videoCommentRouter) | 54 | videosRouter.use('/', videoCommentRouter) |
55 | videosRouter.use('/', studioRouter) | 55 | videosRouter.use('/', studioRouter) |
56 | videosRouter.use('/', videoCaptionsRouter) | 56 | videosRouter.use('/', videoCaptionsRouter) |
57 | videosRouter.use('/', videoImportsRouter) | 57 | videosRouter.use('/', videoImportsRouter) |
58 | videosRouter.use('/', ownershipVideoRouter) | 58 | videosRouter.use('/', ownershipVideoRouter) |
59 | videosRouter.use('/', watchingRouter) | 59 | videosRouter.use('/', viewRouter) |
60 | videosRouter.use('/', liveRouter) | 60 | videosRouter.use('/', liveRouter) |
61 | videosRouter.use('/', uploadRouter) | 61 | videosRouter.use('/', uploadRouter) |
62 | videosRouter.use('/', updateRouter) | 62 | videosRouter.use('/', updateRouter) |
@@ -103,11 +103,6 @@ videosRouter.get('/:id', | |||
103 | asyncMiddleware(checkVideoFollowConstraints), | 103 | asyncMiddleware(checkVideoFollowConstraints), |
104 | getVideo | 104 | getVideo |
105 | ) | 105 | ) |
106 | videosRouter.post('/:id/views', | ||
107 | openapiOperationDoc({ operationId: 'addView' }), | ||
108 | asyncMiddleware(videosCustomGetValidator('only-video')), | ||
109 | asyncMiddleware(viewVideo) | ||
110 | ) | ||
111 | 106 | ||
112 | videosRouter.delete('/:id', | 107 | videosRouter.delete('/:id', |
113 | openapiOperationDoc({ operationId: 'delVideo' }), | 108 | openapiOperationDoc({ operationId: 'delVideo' }), |
@@ -150,22 +145,6 @@ function getVideo (_req: express.Request, res: express.Response) { | |||
150 | return res.json(video.toFormattedDetailsJSON()) | 145 | return res.json(video.toFormattedDetailsJSON()) |
151 | } | 146 | } |
152 | 147 | ||
153 | async function viewVideo (req: express.Request, res: express.Response) { | ||
154 | const video = res.locals.onlyVideo | ||
155 | |||
156 | const ip = req.ip | ||
157 | const success = await VideoViews.Instance.processView({ video, ip }) | ||
158 | |||
159 | if (success) { | ||
160 | const serverActor = await getServerActor() | ||
161 | await sendView(serverActor, video, undefined) | ||
162 | |||
163 | Hooks.runAction('action:api.video.viewed', { video: video, ip, req, res }) | ||
164 | } | ||
165 | |||
166 | return res.status(HttpStatusCode.NO_CONTENT_204).end() | ||
167 | } | ||
168 | |||
169 | async function getVideoDescription (req: express.Request, res: express.Response) { | 148 | async function getVideoDescription (req: express.Request, res: express.Response) { |
170 | const videoInstance = res.locals.videoAll | 149 | const videoInstance = res.locals.videoAll |
171 | 150 | ||
diff --git a/server/controllers/api/videos/stats.ts b/server/controllers/api/videos/stats.ts new file mode 100644 index 000000000..5f8513e9e --- /dev/null +++ b/server/controllers/api/videos/stats.ts | |||
@@ -0,0 +1,66 @@ | |||
1 | import express from 'express' | ||
2 | import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' | ||
3 | import { VideoStatsTimeserieMetric } from '@shared/models' | ||
4 | import { | ||
5 | asyncMiddleware, | ||
6 | authenticate, | ||
7 | videoOverallStatsValidator, | ||
8 | videoRetentionStatsValidator, | ||
9 | videoTimeserieStatsValidator | ||
10 | } from '../../../middlewares' | ||
11 | |||
12 | const statsRouter = express.Router() | ||
13 | |||
14 | statsRouter.get('/:videoId/stats/overall', | ||
15 | authenticate, | ||
16 | asyncMiddleware(videoOverallStatsValidator), | ||
17 | asyncMiddleware(getOverallStats) | ||
18 | ) | ||
19 | |||
20 | statsRouter.get('/:videoId/stats/timeseries/:metric', | ||
21 | authenticate, | ||
22 | asyncMiddleware(videoTimeserieStatsValidator), | ||
23 | asyncMiddleware(getTimeserieStats) | ||
24 | ) | ||
25 | |||
26 | statsRouter.get('/:videoId/stats/retention', | ||
27 | authenticate, | ||
28 | asyncMiddleware(videoRetentionStatsValidator), | ||
29 | asyncMiddleware(getRetentionStats) | ||
30 | ) | ||
31 | |||
32 | // --------------------------------------------------------------------------- | ||
33 | |||
34 | export { | ||
35 | statsRouter | ||
36 | } | ||
37 | |||
38 | // --------------------------------------------------------------------------- | ||
39 | |||
40 | async function getOverallStats (req: express.Request, res: express.Response) { | ||
41 | const video = res.locals.videoAll | ||
42 | |||
43 | const stats = await LocalVideoViewerModel.getOverallStats(video) | ||
44 | |||
45 | return res.json(stats) | ||
46 | } | ||
47 | |||
48 | async function getRetentionStats (req: express.Request, res: express.Response) { | ||
49 | const video = res.locals.videoAll | ||
50 | |||
51 | const stats = await LocalVideoViewerModel.getRetentionStats(video) | ||
52 | |||
53 | return res.json(stats) | ||
54 | } | ||
55 | |||
56 | async function getTimeserieStats (req: express.Request, res: express.Response) { | ||
57 | const video = res.locals.videoAll | ||
58 | const metric = req.params.metric as VideoStatsTimeserieMetric | ||
59 | |||
60 | const stats = await LocalVideoViewerModel.getTimeserieStats({ | ||
61 | video, | ||
62 | metric | ||
63 | }) | ||
64 | |||
65 | return res.json(stats) | ||
66 | } | ||
diff --git a/server/controllers/api/videos/view.ts b/server/controllers/api/videos/view.ts new file mode 100644 index 000000000..e28cf371a --- /dev/null +++ b/server/controllers/api/videos/view.ts | |||
@@ -0,0 +1,68 @@ | |||
1 | import express from 'express' | ||
2 | import { sendView } from '@server/lib/activitypub/send/send-view' | ||
3 | import { Hooks } from '@server/lib/plugins/hooks' | ||
4 | import { VideoViewsManager } from '@server/lib/views/video-views-manager' | ||
5 | import { getServerActor } from '@server/models/application/application' | ||
6 | import { MVideoId } from '@server/types/models' | ||
7 | import { HttpStatusCode, VideoView } from '@shared/models' | ||
8 | import { asyncMiddleware, methodsValidator, openapiOperationDoc, optionalAuthenticate, videoViewValidator } from '../../../middlewares' | ||
9 | import { UserVideoHistoryModel } from '../../../models/user/user-video-history' | ||
10 | |||
11 | const viewRouter = express.Router() | ||
12 | |||
13 | viewRouter.all( | ||
14 | [ '/:videoId/views', '/:videoId/watching' ], | ||
15 | openapiOperationDoc({ operationId: 'addView' }), | ||
16 | methodsValidator([ 'PUT', 'POST' ]), | ||
17 | optionalAuthenticate, | ||
18 | asyncMiddleware(videoViewValidator), | ||
19 | asyncMiddleware(viewVideo) | ||
20 | ) | ||
21 | |||
22 | // --------------------------------------------------------------------------- | ||
23 | |||
24 | export { | ||
25 | viewRouter | ||
26 | } | ||
27 | |||
28 | // --------------------------------------------------------------------------- | ||
29 | |||
30 | async function viewVideo (req: express.Request, res: express.Response) { | ||
31 | const video = res.locals.onlyVideo | ||
32 | |||
33 | const body = req.body as VideoView | ||
34 | |||
35 | const ip = req.ip | ||
36 | const { successView, successViewer } = await VideoViewsManager.Instance.processLocalView({ | ||
37 | video, | ||
38 | ip, | ||
39 | currentTime: body.currentTime, | ||
40 | viewEvent: body.viewEvent | ||
41 | }) | ||
42 | |||
43 | if (successView) { | ||
44 | await sendView({ byActor: await getServerActor(), video, type: 'view' }) | ||
45 | |||
46 | Hooks.runAction('action:api.video.viewed', { video: video, ip, req, res }) | ||
47 | } | ||
48 | |||
49 | if (successViewer) { | ||
50 | await sendView({ byActor: await getServerActor(), video, type: 'viewer' }) | ||
51 | } | ||
52 | |||
53 | await updateUserHistoryIfNeeded(body, video, res) | ||
54 | |||
55 | return res.status(HttpStatusCode.NO_CONTENT_204).end() | ||
56 | } | ||
57 | |||
58 | async function updateUserHistoryIfNeeded (body: VideoView, video: MVideoId, res: express.Response) { | ||
59 | const user = res.locals.oauth?.token.User | ||
60 | if (!user) return | ||
61 | if (user.videosHistoryEnabled !== true) return | ||
62 | |||
63 | await UserVideoHistoryModel.upsert({ | ||
64 | videoId: video.id, | ||
65 | userId: user.id, | ||
66 | currentTime: body.currentTime | ||
67 | }) | ||
68 | } | ||
diff --git a/server/controllers/api/videos/watching.ts b/server/controllers/api/videos/watching.ts deleted file mode 100644 index 3fd22caac..000000000 --- a/server/controllers/api/videos/watching.ts +++ /dev/null | |||
@@ -1,44 +0,0 @@ | |||
1 | import express from 'express' | ||
2 | import { HttpStatusCode, UserWatchingVideo } from '@shared/models' | ||
3 | import { | ||
4 | asyncMiddleware, | ||
5 | asyncRetryTransactionMiddleware, | ||
6 | authenticate, | ||
7 | openapiOperationDoc, | ||
8 | videoWatchingValidator | ||
9 | } from '../../../middlewares' | ||
10 | import { UserVideoHistoryModel } from '../../../models/user/user-video-history' | ||
11 | |||
12 | const watchingRouter = express.Router() | ||
13 | |||
14 | watchingRouter.put('/:videoId/watching', | ||
15 | openapiOperationDoc({ operationId: 'setProgress' }), | ||
16 | authenticate, | ||
17 | asyncMiddleware(videoWatchingValidator), | ||
18 | asyncRetryTransactionMiddleware(userWatchVideo) | ||
19 | ) | ||
20 | |||
21 | // --------------------------------------------------------------------------- | ||
22 | |||
23 | export { | ||
24 | watchingRouter | ||
25 | } | ||
26 | |||
27 | // --------------------------------------------------------------------------- | ||
28 | |||
29 | async function userWatchVideo (req: express.Request, res: express.Response) { | ||
30 | const user = res.locals.oauth.token.User | ||
31 | |||
32 | const body: UserWatchingVideo = req.body | ||
33 | const { id: videoId } = res.locals.videoId | ||
34 | |||
35 | await UserVideoHistoryModel.upsert({ | ||
36 | videoId, | ||
37 | userId: user.id, | ||
38 | currentTime: body.currentTime | ||
39 | }) | ||
40 | |||
41 | return res.type('json') | ||
42 | .status(HttpStatusCode.NO_CONTENT_204) | ||
43 | .end() | ||
44 | } | ||
diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts index b5c96f6e7..90a918523 100644 --- a/server/helpers/custom-validators/activitypub/activity.ts +++ b/server/helpers/custom-validators/activitypub/activity.ts | |||
@@ -8,6 +8,7 @@ import { isActivityPubUrlValid, isBaseActivityValid, isObjectValid } from './mis | |||
8 | import { isPlaylistObjectValid } from './playlist' | 8 | import { isPlaylistObjectValid } from './playlist' |
9 | import { sanitizeAndCheckVideoCommentObject } from './video-comments' | 9 | import { sanitizeAndCheckVideoCommentObject } from './video-comments' |
10 | import { sanitizeAndCheckVideoTorrentObject } from './videos' | 10 | import { sanitizeAndCheckVideoTorrentObject } from './videos' |
11 | import { isWatchActionObjectValid } from './watch-action' | ||
11 | 12 | ||
12 | function isRootActivityValid (activity: any) { | 13 | function isRootActivityValid (activity: any) { |
13 | return isCollection(activity) || isActivity(activity) | 14 | return isCollection(activity) || isActivity(activity) |
@@ -82,6 +83,7 @@ function isCreateActivityValid (activity: any) { | |||
82 | isDislikeActivityValid(activity.object) || | 83 | isDislikeActivityValid(activity.object) || |
83 | isFlagActivityValid(activity.object) || | 84 | isFlagActivityValid(activity.object) || |
84 | isPlaylistObjectValid(activity.object) || | 85 | isPlaylistObjectValid(activity.object) || |
86 | isWatchActionObjectValid(activity.object) || | ||
85 | 87 | ||
86 | isCacheFileObjectValid(activity.object) || | 88 | isCacheFileObjectValid(activity.object) || |
87 | sanitizeAndCheckVideoCommentObject(activity.object) || | 89 | sanitizeAndCheckVideoCommentObject(activity.object) || |
diff --git a/server/helpers/custom-validators/activitypub/misc.ts b/server/helpers/custom-validators/activitypub/misc.ts index 4ee8e6fee..9d823299f 100644 --- a/server/helpers/custom-validators/activitypub/misc.ts +++ b/server/helpers/custom-validators/activitypub/misc.ts | |||
@@ -57,10 +57,19 @@ function setValidAttributedTo (obj: any) { | |||
57 | return true | 57 | return true |
58 | } | 58 | } |
59 | 59 | ||
60 | function isActivityPubVideoDurationValid (value: string) { | ||
61 | // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration | ||
62 | return exists(value) && | ||
63 | typeof value === 'string' && | ||
64 | value.startsWith('PT') && | ||
65 | value.endsWith('S') | ||
66 | } | ||
67 | |||
60 | export { | 68 | export { |
61 | isUrlValid, | 69 | isUrlValid, |
62 | isActivityPubUrlValid, | 70 | isActivityPubUrlValid, |
63 | isBaseActivityValid, | 71 | isBaseActivityValid, |
64 | setValidAttributedTo, | 72 | setValidAttributedTo, |
65 | isObjectValid | 73 | isObjectValid, |
74 | isActivityPubVideoDurationValid | ||
66 | } | 75 | } |
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index 80a321117..2a2f008b9 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts | |||
@@ -4,7 +4,7 @@ import { ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject } from '@s | |||
4 | import { LiveVideoLatencyMode, VideoState } from '../../../../shared/models/videos' | 4 | import { LiveVideoLatencyMode, VideoState } from '../../../../shared/models/videos' |
5 | import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers/constants' | 5 | import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers/constants' |
6 | import { peertubeTruncate } from '../../core-utils' | 6 | import { peertubeTruncate } from '../../core-utils' |
7 | import { exists, isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc' | 7 | import { isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc' |
8 | import { isLiveLatencyModeValid } from '../video-lives' | 8 | import { isLiveLatencyModeValid } from '../video-lives' |
9 | import { | 9 | import { |
10 | isVideoDurationValid, | 10 | isVideoDurationValid, |
@@ -14,22 +14,13 @@ import { | |||
14 | isVideoTruncatedDescriptionValid, | 14 | isVideoTruncatedDescriptionValid, |
15 | isVideoViewsValid | 15 | isVideoViewsValid |
16 | } from '../videos' | 16 | } from '../videos' |
17 | import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc' | 17 | import { isActivityPubUrlValid, isActivityPubVideoDurationValid, isBaseActivityValid, setValidAttributedTo } from './misc' |
18 | 18 | ||
19 | function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { | 19 | function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { |
20 | return isBaseActivityValid(activity, 'Update') && | 20 | return isBaseActivityValid(activity, 'Update') && |
21 | sanitizeAndCheckVideoTorrentObject(activity.object) | 21 | sanitizeAndCheckVideoTorrentObject(activity.object) |
22 | } | 22 | } |
23 | 23 | ||
24 | function isActivityPubVideoDurationValid (value: string) { | ||
25 | // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration | ||
26 | return exists(value) && | ||
27 | typeof value === 'string' && | ||
28 | value.startsWith('PT') && | ||
29 | value.endsWith('S') && | ||
30 | isVideoDurationValid(value.replace(/[^0-9]+/g, '')) | ||
31 | } | ||
32 | |||
33 | function sanitizeAndCheckVideoTorrentObject (video: any) { | 24 | function sanitizeAndCheckVideoTorrentObject (video: any) { |
34 | if (!video || video.type !== 'Video') return false | 25 | if (!video || video.type !== 'Video') return false |
35 | 26 | ||
@@ -71,6 +62,7 @@ function sanitizeAndCheckVideoTorrentObject (video: any) { | |||
71 | return isActivityPubUrlValid(video.id) && | 62 | return isActivityPubUrlValid(video.id) && |
72 | isVideoNameValid(video.name) && | 63 | isVideoNameValid(video.name) && |
73 | isActivityPubVideoDurationValid(video.duration) && | 64 | isActivityPubVideoDurationValid(video.duration) && |
65 | isVideoDurationValid(video.duration.replace(/[^0-9]+/g, '')) && | ||
74 | isUUIDValid(video.uuid) && | 66 | isUUIDValid(video.uuid) && |
75 | (!video.category || isRemoteNumberIdentifierValid(video.category)) && | 67 | (!video.category || isRemoteNumberIdentifierValid(video.category)) && |
76 | (!video.licence || isRemoteNumberIdentifierValid(video.licence)) && | 68 | (!video.licence || isRemoteNumberIdentifierValid(video.licence)) && |
diff --git a/server/helpers/custom-validators/activitypub/watch-action.ts b/server/helpers/custom-validators/activitypub/watch-action.ts new file mode 100644 index 000000000..b9ffa63f6 --- /dev/null +++ b/server/helpers/custom-validators/activitypub/watch-action.ts | |||
@@ -0,0 +1,37 @@ | |||
1 | import { WatchActionObject } from '@shared/models' | ||
2 | import { exists, isDateValid, isUUIDValid } from '../misc' | ||
3 | import { isVideoTimeValid } from '../video-view' | ||
4 | import { isActivityPubVideoDurationValid, isObjectValid } from './misc' | ||
5 | |||
6 | function isWatchActionObjectValid (action: WatchActionObject) { | ||
7 | return exists(action) && | ||
8 | action.type === 'WatchAction' && | ||
9 | isObjectValid(action.id) && | ||
10 | isActivityPubVideoDurationValid(action.duration) && | ||
11 | isDateValid(action.startTime) && | ||
12 | isDateValid(action.endTime) && | ||
13 | isLocationValid(action.location) && | ||
14 | isUUIDValid(action.uuid) && | ||
15 | isObjectValid(action.object) && | ||
16 | isWatchSectionsValid(action.watchSections) | ||
17 | } | ||
18 | |||
19 | // --------------------------------------------------------------------------- | ||
20 | |||
21 | export { | ||
22 | isWatchActionObjectValid | ||
23 | } | ||
24 | |||
25 | // --------------------------------------------------------------------------- | ||
26 | |||
27 | function isLocationValid (location: any) { | ||
28 | if (!location) return true | ||
29 | |||
30 | return typeof location === 'object' && typeof location.addressCountry === 'string' | ||
31 | } | ||
32 | |||
33 | function isWatchSectionsValid (sections: WatchActionObject['watchSections']) { | ||
34 | return Array.isArray(sections) && sections.every(s => { | ||
35 | return isVideoTimeValid(s.startTimestamp) && isVideoTimeValid(s.endTimestamp) | ||
36 | }) | ||
37 | } | ||
diff --git a/server/helpers/custom-validators/video-stats.ts b/server/helpers/custom-validators/video-stats.ts new file mode 100644 index 000000000..1e22f0654 --- /dev/null +++ b/server/helpers/custom-validators/video-stats.ts | |||
@@ -0,0 +1,16 @@ | |||
1 | import { VideoStatsTimeserieMetric } from '@shared/models' | ||
2 | |||
3 | const validMetrics = new Set<VideoStatsTimeserieMetric>([ | ||
4 | 'viewers', | ||
5 | 'aggregateWatchTime' | ||
6 | ]) | ||
7 | |||
8 | function isValidStatTimeserieMetric (value: VideoStatsTimeserieMetric) { | ||
9 | return validMetrics.has(value) | ||
10 | } | ||
11 | |||
12 | // --------------------------------------------------------------------------- | ||
13 | |||
14 | export { | ||
15 | isValidStatTimeserieMetric | ||
16 | } | ||
diff --git a/server/helpers/custom-validators/video-view.ts b/server/helpers/custom-validators/video-view.ts new file mode 100644 index 000000000..091c92083 --- /dev/null +++ b/server/helpers/custom-validators/video-view.ts | |||
@@ -0,0 +1,12 @@ | |||
1 | import { exists } from './misc' | ||
2 | |||
3 | function isVideoTimeValid (value: number, videoDuration?: number) { | ||
4 | if (value < 0) return false | ||
5 | if (exists(videoDuration) && value > videoDuration) return false | ||
6 | |||
7 | return true | ||
8 | } | ||
9 | |||
10 | export { | ||
11 | isVideoTimeValid | ||
12 | } | ||
diff --git a/server/helpers/geo-ip.ts b/server/helpers/geo-ip.ts new file mode 100644 index 000000000..4ba7011c2 --- /dev/null +++ b/server/helpers/geo-ip.ts | |||
@@ -0,0 +1,78 @@ | |||
1 | import { pathExists, writeFile } from 'fs-extra' | ||
2 | import maxmind, { CountryResponse, Reader } from 'maxmind' | ||
3 | import { join } from 'path' | ||
4 | import { CONFIG } from '@server/initializers/config' | ||
5 | import { logger, loggerTagsFactory } from './logger' | ||
6 | import { isBinaryResponse, peertubeGot } from './requests' | ||
7 | |||
8 | const lTags = loggerTagsFactory('geo-ip') | ||
9 | |||
10 | const mmbdFilename = 'dbip-country-lite-latest.mmdb' | ||
11 | const mmdbPath = join(CONFIG.STORAGE.BIN_DIR, mmbdFilename) | ||
12 | |||
13 | export class GeoIP { | ||
14 | private static instance: GeoIP | ||
15 | |||
16 | private reader: Reader<CountryResponse> | ||
17 | |||
18 | private constructor () { | ||
19 | } | ||
20 | |||
21 | async safeCountryISOLookup (ip: string): Promise<string> { | ||
22 | if (CONFIG.GEO_IP.ENABLED === false) return null | ||
23 | |||
24 | await this.initReaderIfNeeded() | ||
25 | |||
26 | try { | ||
27 | const result = this.reader.get(ip) | ||
28 | if (!result) return null | ||
29 | |||
30 | return result.country.iso_code | ||
31 | } catch (err) { | ||
32 | logger.error('Cannot get country from IP.', { err }) | ||
33 | |||
34 | return null | ||
35 | } | ||
36 | } | ||
37 | |||
38 | async updateDatabase () { | ||
39 | if (CONFIG.GEO_IP.ENABLED === false) return | ||
40 | |||
41 | const url = CONFIG.GEO_IP.COUNTRY.DATABASE_URL | ||
42 | |||
43 | logger.info('Updating GeoIP database from %s.', url, lTags()) | ||
44 | |||
45 | const gotOptions = { context: { bodyKBLimit: 200_000 }, responseType: 'buffer' as 'buffer' } | ||
46 | |||
47 | try { | ||
48 | const gotResult = await peertubeGot(url, gotOptions) | ||
49 | |||
50 | if (!isBinaryResponse(gotResult)) { | ||
51 | throw new Error('Not a binary response') | ||
52 | } | ||
53 | |||
54 | await writeFile(mmdbPath, gotResult.body) | ||
55 | |||
56 | // Reini reader | ||
57 | this.reader = undefined | ||
58 | |||
59 | logger.info('GeoIP database updated %s.', mmdbPath, lTags()) | ||
60 | } catch (err) { | ||
61 | logger.error('Cannot update GeoIP database from %s.', url, { err, ...lTags() }) | ||
62 | } | ||
63 | } | ||
64 | |||
65 | private async initReaderIfNeeded () { | ||
66 | if (!this.reader) { | ||
67 | if (!await pathExists(mmdbPath)) { | ||
68 | await this.updateDatabase() | ||
69 | } | ||
70 | |||
71 | this.reader = await maxmind.open(mmdbPath) | ||
72 | } | ||
73 | } | ||
74 | |||
75 | static get Instance () { | ||
76 | return this.instance || (this.instance = new this()) | ||
77 | } | ||
78 | } | ||
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index 0f23a2d73..f2ef3d567 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts | |||
@@ -44,6 +44,7 @@ function checkMissedConfig () { | |||
44 | 'history.videos.max_age', 'views.videos.remote.max_age', 'views.videos.local_buffer_update_interval', 'views.videos.ip_view_expiration', | 44 | 'history.videos.max_age', 'views.videos.remote.max_age', 'views.videos.local_buffer_update_interval', 'views.videos.ip_view_expiration', |
45 | 'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max', | 45 | 'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max', |
46 | 'theme.default', | 46 | 'theme.default', |
47 | 'geo_ip.enabled', 'geo_ip.country.database_url', | ||
47 | 'remote_redundancy.videos.accept_from', | 48 | 'remote_redundancy.videos.accept_from', |
48 | 'federation.videos.federate_unlisted', 'federation.videos.cleanup_remote_interactions', | 49 | 'federation.videos.federate_unlisted', 'federation.videos.cleanup_remote_interactions', |
49 | 'peertube.check_latest_version.enabled', 'peertube.check_latest_version.url', | 50 | 'peertube.check_latest_version.enabled', 'peertube.check_latest_version.url', |
diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 122cb9472..d8f5f3496 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts | |||
@@ -215,6 +215,12 @@ const CONFIG = { | |||
215 | IP_VIEW_EXPIRATION: parseDurationToMs(config.get('views.videos.ip_view_expiration')) | 215 | IP_VIEW_EXPIRATION: parseDurationToMs(config.get('views.videos.ip_view_expiration')) |
216 | } | 216 | } |
217 | }, | 217 | }, |
218 | GEO_IP: { | ||
219 | ENABLED: config.get<boolean>('geo_ip.enabled'), | ||
220 | COUNTRY: { | ||
221 | DATABASE_URL: config.get<string>('geo_ip.country.database_url') | ||
222 | } | ||
223 | }, | ||
218 | PLUGINS: { | 224 | PLUGINS: { |
219 | INDEX: { | 225 | INDEX: { |
220 | ENABLED: config.get<boolean>('plugins.index.enabled'), | 226 | ENABLED: config.get<boolean>('plugins.index.enabled'), |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 6bcefe0db..4929923dc 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' | |||
24 | 24 | ||
25 | // --------------------------------------------------------------------------- | 25 | // --------------------------------------------------------------------------- |
26 | 26 | ||
27 | const LAST_MIGRATION_VERSION = 700 | 27 | const LAST_MIGRATION_VERSION = 705 |
28 | 28 | ||
29 | // --------------------------------------------------------------------------- | 29 | // --------------------------------------------------------------------------- |
30 | 30 | ||
@@ -228,6 +228,7 @@ const SCHEDULER_INTERVALS_MS = { | |||
228 | REMOVE_OLD_JOBS: 60000 * 60, // 1 hour | 228 | REMOVE_OLD_JOBS: 60000 * 60, // 1 hour |
229 | UPDATE_VIDEOS: 60000, // 1 minute | 229 | UPDATE_VIDEOS: 60000, // 1 minute |
230 | YOUTUBE_DL_UPDATE: 60000 * 60 * 24, // 1 day | 230 | YOUTUBE_DL_UPDATE: 60000 * 60 * 24, // 1 day |
231 | GEO_IP_UPDATE: 60000 * 60 * 24, // 1 day | ||
231 | VIDEO_VIEWS_BUFFER_UPDATE: CONFIG.VIEWS.VIDEOS.LOCAL_BUFFER_UPDATE_INTERVAL, | 232 | VIDEO_VIEWS_BUFFER_UPDATE: CONFIG.VIEWS.VIDEOS.LOCAL_BUFFER_UPDATE_INTERVAL, |
232 | CHECK_PLUGINS: CONFIG.PLUGINS.INDEX.CHECK_LATEST_VERSIONS_INTERVAL, | 233 | CHECK_PLUGINS: CONFIG.PLUGINS.INDEX.CHECK_LATEST_VERSIONS_INTERVAL, |
233 | CHECK_PEERTUBE_VERSION: 60000 * 60 * 24, // 1 day | 234 | CHECK_PEERTUBE_VERSION: 60000 * 60 * 24, // 1 day |
@@ -366,9 +367,12 @@ const CONSTRAINTS_FIELDS = { | |||
366 | 367 | ||
367 | const VIEW_LIFETIME = { | 368 | const VIEW_LIFETIME = { |
368 | VIEW: CONFIG.VIEWS.VIDEOS.IP_VIEW_EXPIRATION, | 369 | VIEW: CONFIG.VIEWS.VIDEOS.IP_VIEW_EXPIRATION, |
369 | VIEWER: 60000 * 5 // 5 minutes | 370 | VIEWER: 60000 * 5, // 5 minutes |
371 | VIEWER_STATS: 60000 * 60 // 1 hour | ||
370 | } | 372 | } |
371 | 373 | ||
374 | const MAX_LOCAL_VIEWER_WATCH_SECTIONS = 10 | ||
375 | |||
372 | let CONTACT_FORM_LIFETIME = 60000 * 60 // 1 hour | 376 | let CONTACT_FORM_LIFETIME = 60000 * 60 // 1 hour |
373 | 377 | ||
374 | const VIDEO_TRANSCODING_FPS: VideoTranscodingFPS = { | 378 | const VIDEO_TRANSCODING_FPS: VideoTranscodingFPS = { |
@@ -800,6 +804,12 @@ const SEARCH_INDEX = { | |||
800 | 804 | ||
801 | // --------------------------------------------------------------------------- | 805 | // --------------------------------------------------------------------------- |
802 | 806 | ||
807 | const STATS_TIMESERIE = { | ||
808 | MAX_DAYS: 30 | ||
809 | } | ||
810 | |||
811 | // --------------------------------------------------------------------------- | ||
812 | |||
803 | // Special constants for a test instance | 813 | // Special constants for a test instance |
804 | if (isTestInstance() === true) { | 814 | if (isTestInstance() === true) { |
805 | PRIVATE_RSA_KEY_SIZE = 1024 | 815 | PRIVATE_RSA_KEY_SIZE = 1024 |
@@ -836,6 +846,7 @@ if (isTestInstance() === true) { | |||
836 | REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1 | 846 | REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1 |
837 | 847 | ||
838 | VIEW_LIFETIME.VIEWER = 1000 * 5 // 5 second | 848 | VIEW_LIFETIME.VIEWER = 1000 * 5 // 5 second |
849 | VIEW_LIFETIME.VIEWER_STATS = 1000 * 5 // 5 second | ||
839 | CONTACT_FORM_LIFETIME = 1000 // 1 second | 850 | CONTACT_FORM_LIFETIME = 1000 // 1 second |
840 | 851 | ||
841 | JOB_ATTEMPTS['email'] = 1 | 852 | JOB_ATTEMPTS['email'] = 1 |
@@ -907,6 +918,7 @@ export { | |||
907 | LAST_MIGRATION_VERSION, | 918 | LAST_MIGRATION_VERSION, |
908 | OAUTH_LIFETIME, | 919 | OAUTH_LIFETIME, |
909 | CUSTOM_HTML_TAG_COMMENTS, | 920 | CUSTOM_HTML_TAG_COMMENTS, |
921 | STATS_TIMESERIE, | ||
910 | BROADCAST_CONCURRENCY, | 922 | BROADCAST_CONCURRENCY, |
911 | AUDIT_LOG_FILENAME, | 923 | AUDIT_LOG_FILENAME, |
912 | PAGINATION, | 924 | PAGINATION, |
@@ -949,6 +961,7 @@ export { | |||
949 | ABUSE_STATES, | 961 | ABUSE_STATES, |
950 | LRU_CACHE, | 962 | LRU_CACHE, |
951 | REQUEST_TIMEOUTS, | 963 | REQUEST_TIMEOUTS, |
964 | MAX_LOCAL_VIEWER_WATCH_SECTIONS, | ||
952 | USER_PASSWORD_RESET_LIFETIME, | 965 | USER_PASSWORD_RESET_LIFETIME, |
953 | USER_PASSWORD_CREATE_LIFETIME, | 966 | USER_PASSWORD_CREATE_LIFETIME, |
954 | MEMOIZE_TTL, | 967 | MEMOIZE_TTL, |
diff --git a/server/initializers/database.ts b/server/initializers/database.ts index 0e690f6ae..7a7ba61f4 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts | |||
@@ -1,10 +1,14 @@ | |||
1 | import { QueryTypes, Transaction } from 'sequelize' | 1 | import { QueryTypes, Transaction } from 'sequelize' |
2 | import { Sequelize as SequelizeTypescript } from 'sequelize-typescript' | 2 | import { Sequelize as SequelizeTypescript } from 'sequelize-typescript' |
3 | import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' | ||
3 | import { TrackerModel } from '@server/models/server/tracker' | 4 | import { TrackerModel } from '@server/models/server/tracker' |
4 | import { VideoTrackerModel } from '@server/models/server/video-tracker' | 5 | import { VideoTrackerModel } from '@server/models/server/video-tracker' |
5 | import { UserModel } from '@server/models/user/user' | 6 | import { UserModel } from '@server/models/user/user' |
6 | import { UserNotificationModel } from '@server/models/user/user-notification' | 7 | import { UserNotificationModel } from '@server/models/user/user-notification' |
7 | import { UserVideoHistoryModel } from '@server/models/user/user-video-history' | 8 | import { UserVideoHistoryModel } from '@server/models/user/user-video-history' |
9 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
10 | import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' | ||
11 | import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' | ||
8 | import { isTestInstance } from '../helpers/core-utils' | 12 | import { isTestInstance } from '../helpers/core-utils' |
9 | import { logger } from '../helpers/logger' | 13 | import { logger } from '../helpers/logger' |
10 | import { AbuseModel } from '../models/abuse/abuse' | 14 | import { AbuseModel } from '../models/abuse/abuse' |
@@ -42,10 +46,8 @@ import { VideoPlaylistElementModel } from '../models/video/video-playlist-elemen | |||
42 | import { VideoShareModel } from '../models/video/video-share' | 46 | import { VideoShareModel } from '../models/video/video-share' |
43 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' | 47 | import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' |
44 | import { VideoTagModel } from '../models/video/video-tag' | 48 | import { VideoTagModel } from '../models/video/video-tag' |
45 | import { VideoViewModel } from '../models/video/video-view' | 49 | import { VideoViewModel } from '../models/view/video-view' |
46 | import { CONFIG } from './config' | 50 | import { CONFIG } from './config' |
47 | import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' | ||
48 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | ||
49 | 51 | ||
50 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string | 52 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string |
51 | 53 | ||
@@ -140,6 +142,8 @@ async function initDatabaseModels (silent: boolean) { | |||
140 | VideoStreamingPlaylistModel, | 142 | VideoStreamingPlaylistModel, |
141 | VideoPlaylistModel, | 143 | VideoPlaylistModel, |
142 | VideoPlaylistElementModel, | 144 | VideoPlaylistElementModel, |
145 | LocalVideoViewerModel, | ||
146 | LocalVideoViewerWatchSectionModel, | ||
143 | ThumbnailModel, | 147 | ThumbnailModel, |
144 | TrackerModel, | 148 | TrackerModel, |
145 | VideoTrackerModel, | 149 | VideoTrackerModel, |
diff --git a/server/initializers/migrations/0705-local-video-viewers.ts b/server/initializers/migrations/0705-local-video-viewers.ts new file mode 100644 index 000000000..123402641 --- /dev/null +++ b/server/initializers/migrations/0705-local-video-viewers.ts | |||
@@ -0,0 +1,52 @@ | |||
1 | import * as Sequelize from 'sequelize' | ||
2 | |||
3 | async function up (utils: { | ||
4 | transaction: Sequelize.Transaction | ||
5 | queryInterface: Sequelize.QueryInterface | ||
6 | sequelize: Sequelize.Sequelize | ||
7 | db: any | ||
8 | }): Promise<void> { | ||
9 | const { transaction } = utils | ||
10 | |||
11 | { | ||
12 | const query = ` | ||
13 | CREATE TABLE IF NOT EXISTS "localVideoViewer" ( | ||
14 | "id" serial, | ||
15 | "startDate" timestamp with time zone NOT NULL, | ||
16 | "endDate" timestamp with time zone NOT NULL, | ||
17 | "watchTime" integer NOT NULL, | ||
18 | "country" varchar(255), | ||
19 | "uuid" uuid NOT NULL, | ||
20 | "url" varchar(255) NOT NULL, | ||
21 | "videoId" integer NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE, | ||
22 | "createdAt" timestamp with time zone NOT NULL, | ||
23 | PRIMARY KEY ("id") | ||
24 | ); | ||
25 | ` | ||
26 | await utils.sequelize.query(query, { transaction }) | ||
27 | } | ||
28 | |||
29 | { | ||
30 | const query = ` | ||
31 | CREATE TABLE IF NOT EXISTS "localVideoViewerWatchSection" ( | ||
32 | "id" serial, | ||
33 | "watchStart" integer NOT NULL, | ||
34 | "watchEnd" integer NOT NULL, | ||
35 | "localVideoViewerId" integer NOT NULL REFERENCES "localVideoViewer" ("id") ON DELETE CASCADE ON UPDATE CASCADE, | ||
36 | "createdAt" timestamp with time zone NOT NULL, | ||
37 | PRIMARY KEY ("id") | ||
38 | ); | ||
39 | ` | ||
40 | await utils.sequelize.query(query, { transaction }) | ||
41 | } | ||
42 | |||
43 | } | ||
44 | |||
45 | function down () { | ||
46 | throw new Error('Not implemented.') | ||
47 | } | ||
48 | |||
49 | export { | ||
50 | up, | ||
51 | down | ||
52 | } | ||
diff --git a/server/lib/activitypub/activity.ts b/server/lib/activitypub/activity.ts index cccb7b1c1..e6cec1ba7 100644 --- a/server/lib/activitypub/activity.ts +++ b/server/lib/activitypub/activity.ts | |||
@@ -4,6 +4,17 @@ function getAPId (object: string | { id: string }) { | |||
4 | return object.id | 4 | return object.id |
5 | } | 5 | } |
6 | 6 | ||
7 | function getActivityStreamDuration (duration: number) { | ||
8 | // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration | ||
9 | return 'PT' + duration + 'S' | ||
10 | } | ||
11 | |||
12 | function getDurationFromActivityStream (duration: string) { | ||
13 | return parseInt(duration.replace(/[^\d]+/, '')) | ||
14 | } | ||
15 | |||
7 | export { | 16 | export { |
8 | getAPId | 17 | getAPId, |
18 | getActivityStreamDuration, | ||
19 | getDurationFromActivityStream | ||
9 | } | 20 | } |
diff --git a/server/lib/activitypub/context.ts b/server/lib/activitypub/context.ts index 3bc40e2aa..b452cf9b3 100644 --- a/server/lib/activitypub/context.ts +++ b/server/lib/activitypub/context.ts | |||
@@ -15,7 +15,7 @@ export { | |||
15 | 15 | ||
16 | type ContextValue = { [ id: string ]: (string | { '@type': string, '@id': string }) } | 16 | type ContextValue = { [ id: string ]: (string | { '@type': string, '@id': string }) } |
17 | 17 | ||
18 | const contextStore = { | 18 | const contextStore: { [ id in ContextType ]: (string | { [ id: string ]: string })[] } = { |
19 | Video: buildContext({ | 19 | Video: buildContext({ |
20 | Hashtag: 'as:Hashtag', | 20 | Hashtag: 'as:Hashtag', |
21 | uuid: 'sc:identifier', | 21 | uuid: 'sc:identifier', |
@@ -109,7 +109,8 @@ const contextStore = { | |||
109 | stopTimestamp: { | 109 | stopTimestamp: { |
110 | '@type': 'sc:Number', | 110 | '@type': 'sc:Number', |
111 | '@id': 'pt:stopTimestamp' | 111 | '@id': 'pt:stopTimestamp' |
112 | } | 112 | }, |
113 | uuid: 'sc:identifier' | ||
113 | }), | 114 | }), |
114 | 115 | ||
115 | CacheFile: buildContext({ | 116 | CacheFile: buildContext({ |
@@ -128,6 +129,24 @@ const contextStore = { | |||
128 | } | 129 | } |
129 | }), | 130 | }), |
130 | 131 | ||
132 | WatchAction: buildContext({ | ||
133 | WatchAction: 'sc:WatchAction', | ||
134 | startTimestamp: { | ||
135 | '@type': 'sc:Number', | ||
136 | '@id': 'pt:startTimestamp' | ||
137 | }, | ||
138 | stopTimestamp: { | ||
139 | '@type': 'sc:Number', | ||
140 | '@id': 'pt:stopTimestamp' | ||
141 | }, | ||
142 | watchSection: { | ||
143 | '@type': 'sc:Number', | ||
144 | '@id': 'pt:stopTimestamp' | ||
145 | }, | ||
146 | uuid: 'sc:identifier' | ||
147 | }), | ||
148 | |||
149 | Collection: buildContext(), | ||
131 | Follow: buildContext(), | 150 | Follow: buildContext(), |
132 | Reject: buildContext(), | 151 | Reject: buildContext(), |
133 | Accept: buildContext(), | 152 | Accept: buildContext(), |
diff --git a/server/lib/activitypub/local-video-viewer.ts b/server/lib/activitypub/local-video-viewer.ts new file mode 100644 index 000000000..738083adc --- /dev/null +++ b/server/lib/activitypub/local-video-viewer.ts | |||
@@ -0,0 +1,42 @@ | |||
1 | import { Transaction } from 'sequelize' | ||
2 | import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' | ||
3 | import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' | ||
4 | import { MVideo } from '@server/types/models' | ||
5 | import { WatchActionObject } from '@shared/models' | ||
6 | import { getDurationFromActivityStream } from './activity' | ||
7 | |||
8 | async function createOrUpdateLocalVideoViewer (watchAction: WatchActionObject, video: MVideo, t: Transaction) { | ||
9 | const stats = await LocalVideoViewerModel.loadByUrl(watchAction.id) | ||
10 | if (stats) await stats.destroy({ transaction: t }) | ||
11 | |||
12 | const localVideoViewer = await LocalVideoViewerModel.create({ | ||
13 | url: watchAction.id, | ||
14 | uuid: watchAction.uuid, | ||
15 | |||
16 | watchTime: getDurationFromActivityStream(watchAction.duration), | ||
17 | |||
18 | startDate: new Date(watchAction.startTime), | ||
19 | endDate: new Date(watchAction.endTime), | ||
20 | |||
21 | country: watchAction.location | ||
22 | ? watchAction.location.addressCountry | ||
23 | : null, | ||
24 | |||
25 | videoId: video.id | ||
26 | }) | ||
27 | |||
28 | await LocalVideoViewerWatchSectionModel.bulkCreateSections({ | ||
29 | localVideoViewerId: localVideoViewer.id, | ||
30 | |||
31 | watchSections: watchAction.watchSections.map(s => ({ | ||
32 | start: s.startTimestamp, | ||
33 | end: s.endTimestamp | ||
34 | })) | ||
35 | }) | ||
36 | } | ||
37 | |||
38 | // --------------------------------------------------------------------------- | ||
39 | |||
40 | export { | ||
41 | createOrUpdateLocalVideoViewer | ||
42 | } | ||
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index b5b1a0feb..3e7931bb2 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts | |||
@@ -1,6 +1,7 @@ | |||
1 | import { isBlockedByServerOrAccount } from '@server/lib/blocklist' | 1 | import { isBlockedByServerOrAccount } from '@server/lib/blocklist' |
2 | import { isRedundancyAccepted } from '@server/lib/redundancy' | 2 | import { isRedundancyAccepted } from '@server/lib/redundancy' |
3 | import { ActivityCreate, CacheFileObject, PlaylistObject, VideoCommentObject, VideoObject } from '@shared/models' | 3 | import { VideoModel } from '@server/models/video/video' |
4 | import { ActivityCreate, CacheFileObject, PlaylistObject, VideoCommentObject, VideoObject, WatchActionObject } from '@shared/models' | ||
4 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | 5 | import { retryTransactionWrapper } from '../../../helpers/database-utils' |
5 | import { logger } from '../../../helpers/logger' | 6 | import { logger } from '../../../helpers/logger' |
6 | import { sequelizeTypescript } from '../../../initializers/database' | 7 | import { sequelizeTypescript } from '../../../initializers/database' |
@@ -8,6 +9,7 @@ import { APProcessorOptions } from '../../../types/activitypub-processor.model' | |||
8 | import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models' | 9 | import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models' |
9 | import { Notifier } from '../../notifier' | 10 | import { Notifier } from '../../notifier' |
10 | import { createOrUpdateCacheFile } from '../cache-file' | 11 | import { createOrUpdateCacheFile } from '../cache-file' |
12 | import { createOrUpdateLocalVideoViewer } from '../local-video-viewer' | ||
11 | import { createOrUpdateVideoPlaylist } from '../playlists' | 13 | import { createOrUpdateVideoPlaylist } from '../playlists' |
12 | import { forwardVideoRelatedActivity } from '../send/shared/send-utils' | 14 | import { forwardVideoRelatedActivity } from '../send/shared/send-utils' |
13 | import { resolveThread } from '../video-comments' | 15 | import { resolveThread } from '../video-comments' |
@@ -32,6 +34,10 @@ async function processCreateActivity (options: APProcessorOptions<ActivityCreate | |||
32 | return retryTransactionWrapper(processCreateVideoComment, activity, byActor, notify) | 34 | return retryTransactionWrapper(processCreateVideoComment, activity, byActor, notify) |
33 | } | 35 | } |
34 | 36 | ||
37 | if (activityType === 'WatchAction') { | ||
38 | return retryTransactionWrapper(processCreateWatchAction, activity) | ||
39 | } | ||
40 | |||
35 | if (activityType === 'CacheFile') { | 41 | if (activityType === 'CacheFile') { |
36 | return retryTransactionWrapper(processCreateCacheFile, activity, byActor) | 42 | return retryTransactionWrapper(processCreateCacheFile, activity, byActor) |
37 | } | 43 | } |
@@ -81,6 +87,19 @@ async function processCreateCacheFile (activity: ActivityCreate, byActor: MActor | |||
81 | } | 87 | } |
82 | } | 88 | } |
83 | 89 | ||
90 | async function processCreateWatchAction (activity: ActivityCreate) { | ||
91 | const watchAction = activity.object as WatchActionObject | ||
92 | |||
93 | if (watchAction.actionStatus !== 'CompletedActionStatus') return | ||
94 | |||
95 | const video = await VideoModel.loadByUrl(watchAction.object) | ||
96 | if (video.remote) return | ||
97 | |||
98 | await sequelizeTypescript.transaction(async t => { | ||
99 | return createOrUpdateLocalVideoViewer(watchAction, video, t) | ||
100 | }) | ||
101 | } | ||
102 | |||
84 | async function processCreateVideoComment (activity: ActivityCreate, byActor: MActorSignature, notify: boolean) { | 103 | async function processCreateVideoComment (activity: ActivityCreate, byActor: MActorSignature, notify: boolean) { |
85 | const commentObject = activity.object as VideoCommentObject | 104 | const commentObject = activity.object as VideoCommentObject |
86 | const byAccount = byActor.Account | 105 | const byAccount = byActor.Account |
diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts index c59940164..bad079843 100644 --- a/server/lib/activitypub/process/process-view.ts +++ b/server/lib/activitypub/process/process-view.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { VideoViews } from '@server/lib/video-views' | 1 | import { VideoViewsManager } from '@server/lib/views/video-views-manager' |
2 | import { ActivityView } from '../../../../shared/models/activitypub' | 2 | import { ActivityView } from '../../../../shared/models/activitypub' |
3 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | 3 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' |
4 | import { MActorSignature } from '../../../types/models' | 4 | import { MActorSignature } from '../../../types/models' |
@@ -32,7 +32,7 @@ async function processCreateView (activity: ActivityView, byActor: MActorSignatu | |||
32 | ? new Date(activity.expires) | 32 | ? new Date(activity.expires) |
33 | : undefined | 33 | : undefined |
34 | 34 | ||
35 | await VideoViews.Instance.processView({ video, ip: null, viewerExpires }) | 35 | await VideoViewsManager.Instance.processRemoteView({ video, viewerExpires }) |
36 | 36 | ||
37 | if (video.isOwned()) { | 37 | if (video.isOwned()) { |
38 | // Forward the view but don't resend the activity to the sender | 38 | // Forward the view but don't resend the activity to the sender |
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts index 5d8763495..7c3a6bdd0 100644 --- a/server/lib/activitypub/send/send-create.ts +++ b/server/lib/activitypub/send/send-create.ts | |||
@@ -6,6 +6,7 @@ import { VideoCommentModel } from '../../../models/video/video-comment' | |||
6 | import { | 6 | import { |
7 | MActorLight, | 7 | MActorLight, |
8 | MCommentOwnerVideo, | 8 | MCommentOwnerVideo, |
9 | MLocalVideoViewerWithWatchSections, | ||
9 | MVideoAccountLight, | 10 | MVideoAccountLight, |
10 | MVideoAP, | 11 | MVideoAP, |
11 | MVideoPlaylistFull, | 12 | MVideoPlaylistFull, |
@@ -19,6 +20,7 @@ import { | |||
19 | getActorsInvolvedInVideo, | 20 | getActorsInvolvedInVideo, |
20 | getAudienceFromFollowersOf, | 21 | getAudienceFromFollowersOf, |
21 | getVideoCommentAudience, | 22 | getVideoCommentAudience, |
23 | sendVideoActivityToOrigin, | ||
22 | sendVideoRelatedActivity, | 24 | sendVideoRelatedActivity, |
23 | unicastTo | 25 | unicastTo |
24 | } from './shared' | 26 | } from './shared' |
@@ -61,6 +63,18 @@ async function sendCreateCacheFile ( | |||
61 | }) | 63 | }) |
62 | } | 64 | } |
63 | 65 | ||
66 | async function sendCreateWatchAction (stats: MLocalVideoViewerWithWatchSections, transaction: Transaction) { | ||
67 | logger.info('Creating job to send create watch action %s.', stats.url, lTags(stats.uuid)) | ||
68 | |||
69 | const byActor = await getServerActor() | ||
70 | |||
71 | const activityBuilder = (audience: ActivityAudience) => { | ||
72 | return buildCreateActivity(stats.url, byActor, stats.toActivityPubObject(), audience) | ||
73 | } | ||
74 | |||
75 | return sendVideoActivityToOrigin(activityBuilder, { byActor, video: stats.Video, transaction, contextType: 'WatchAction' }) | ||
76 | } | ||
77 | |||
64 | async function sendCreateVideoPlaylist (playlist: MVideoPlaylistFull, transaction: Transaction) { | 78 | async function sendCreateVideoPlaylist (playlist: MVideoPlaylistFull, transaction: Transaction) { |
65 | if (playlist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined | 79 | if (playlist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined |
66 | 80 | ||
@@ -175,7 +189,8 @@ export { | |||
175 | buildCreateActivity, | 189 | buildCreateActivity, |
176 | sendCreateVideoComment, | 190 | sendCreateVideoComment, |
177 | sendCreateVideoPlaylist, | 191 | sendCreateVideoPlaylist, |
178 | sendCreateCacheFile | 192 | sendCreateCacheFile, |
193 | sendCreateWatchAction | ||
179 | } | 194 | } |
180 | 195 | ||
181 | // --------------------------------------------------------------------------- | 196 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/activitypub/send/send-view.ts b/server/lib/activitypub/send/send-view.ts index 1f97307b9..1088bf258 100644 --- a/server/lib/activitypub/send/send-view.ts +++ b/server/lib/activitypub/send/send-view.ts | |||
@@ -1,27 +1,49 @@ | |||
1 | import { Transaction } from 'sequelize' | 1 | import { Transaction } from 'sequelize' |
2 | import { VideoViews } from '@server/lib/video-views' | 2 | import { VideoViewsManager } from '@server/lib/views/video-views-manager' |
3 | import { MActorAudience, MVideoImmutable, MVideoUrl } from '@server/types/models' | 3 | import { MActorAudience, MActorLight, MVideoImmutable, MVideoUrl } from '@server/types/models' |
4 | import { ActivityAudience, ActivityView } from '@shared/models' | 4 | import { ActivityAudience, ActivityView } from '@shared/models' |
5 | import { logger } from '../../../helpers/logger' | 5 | import { logger } from '../../../helpers/logger' |
6 | import { ActorModel } from '../../../models/actor/actor' | ||
7 | import { audiencify, getAudience } from '../audience' | 6 | import { audiencify, getAudience } from '../audience' |
8 | import { getLocalVideoViewActivityPubUrl } from '../url' | 7 | import { getLocalVideoViewActivityPubUrl } from '../url' |
9 | import { sendVideoRelatedActivity } from './shared/send-utils' | 8 | import { sendVideoRelatedActivity } from './shared/send-utils' |
10 | 9 | ||
11 | async function sendView (byActor: ActorModel, video: MVideoImmutable, t: Transaction) { | 10 | type ViewType = 'view' | 'viewer' |
12 | logger.info('Creating job to send view of %s.', video.url) | 11 | |
12 | async function sendView (options: { | ||
13 | byActor: MActorLight | ||
14 | type: ViewType | ||
15 | video: MVideoImmutable | ||
16 | transaction?: Transaction | ||
17 | }) { | ||
18 | const { byActor, type, video, transaction } = options | ||
19 | |||
20 | logger.info('Creating job to send %s of %s.', type, video.url) | ||
13 | 21 | ||
14 | const activityBuilder = (audience: ActivityAudience) => { | 22 | const activityBuilder = (audience: ActivityAudience) => { |
15 | const url = getLocalVideoViewActivityPubUrl(byActor, video) | 23 | const url = getLocalVideoViewActivityPubUrl(byActor, video) |
16 | 24 | ||
17 | return buildViewActivity(url, byActor, video, audience) | 25 | return buildViewActivity({ url, byActor, video, audience, type }) |
18 | } | 26 | } |
19 | 27 | ||
20 | return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t, contextType: 'View' }) | 28 | return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction, contextType: 'View' }) |
21 | } | 29 | } |
22 | 30 | ||
23 | function buildViewActivity (url: string, byActor: MActorAudience, video: MVideoUrl, audience?: ActivityAudience): ActivityView { | 31 | // --------------------------------------------------------------------------- |
24 | if (!audience) audience = getAudience(byActor) | 32 | |
33 | export { | ||
34 | sendView | ||
35 | } | ||
36 | |||
37 | // --------------------------------------------------------------------------- | ||
38 | |||
39 | function buildViewActivity (options: { | ||
40 | url: string | ||
41 | byActor: MActorAudience | ||
42 | video: MVideoUrl | ||
43 | type: ViewType | ||
44 | audience?: ActivityAudience | ||
45 | }): ActivityView { | ||
46 | const { url, byActor, type, video, audience = getAudience(byActor) } = options | ||
25 | 47 | ||
26 | return audiencify( | 48 | return audiencify( |
27 | { | 49 | { |
@@ -29,14 +51,11 @@ function buildViewActivity (url: string, byActor: MActorAudience, video: MVideoU | |||
29 | type: 'View' as 'View', | 51 | type: 'View' as 'View', |
30 | actor: byActor.url, | 52 | actor: byActor.url, |
31 | object: video.url, | 53 | object: video.url, |
32 | expires: new Date(VideoViews.Instance.buildViewerExpireTime()).toISOString() | 54 | |
55 | expires: type === 'viewer' | ||
56 | ? new Date(VideoViewsManager.Instance.buildViewerExpireTime()).toISOString() | ||
57 | : undefined | ||
33 | }, | 58 | }, |
34 | audience | 59 | audience |
35 | ) | 60 | ) |
36 | } | 61 | } |
37 | |||
38 | // --------------------------------------------------------------------------- | ||
39 | |||
40 | export { | ||
41 | sendView | ||
42 | } | ||
diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts index 50be4fac9..8443fef4c 100644 --- a/server/lib/activitypub/url.ts +++ b/server/lib/activitypub/url.ts | |||
@@ -7,6 +7,7 @@ import { | |||
7 | MActorId, | 7 | MActorId, |
8 | MActorUrl, | 8 | MActorUrl, |
9 | MCommentId, | 9 | MCommentId, |
10 | MLocalVideoViewer, | ||
10 | MVideoId, | 11 | MVideoId, |
11 | MVideoPlaylistElement, | 12 | MVideoPlaylistElement, |
12 | MVideoUrl, | 13 | MVideoUrl, |
@@ -59,6 +60,10 @@ function getLocalVideoViewActivityPubUrl (byActor: MActorUrl, video: MVideoId) { | |||
59 | return byActor.url + '/views/videos/' + video.id + '/' + new Date().toISOString() | 60 | return byActor.url + '/views/videos/' + video.id + '/' + new Date().toISOString() |
60 | } | 61 | } |
61 | 62 | ||
63 | function getLocalVideoViewerActivityPubUrl (stats: MLocalVideoViewer) { | ||
64 | return WEBSERVER.URL + '/videos/local-viewer/' + stats.uuid | ||
65 | } | ||
66 | |||
62 | function getVideoLikeActivityPubUrlByLocalActor (byActor: MActorUrl, video: MVideoId) { | 67 | function getVideoLikeActivityPubUrlByLocalActor (byActor: MActorUrl, video: MVideoId) { |
63 | return byActor.url + '/likes/' + video.id | 68 | return byActor.url + '/likes/' + video.id |
64 | } | 69 | } |
@@ -167,6 +172,7 @@ export { | |||
167 | getLocalVideoCommentsActivityPubUrl, | 172 | getLocalVideoCommentsActivityPubUrl, |
168 | getLocalVideoLikesActivityPubUrl, | 173 | getLocalVideoLikesActivityPubUrl, |
169 | getLocalVideoDislikesActivityPubUrl, | 174 | getLocalVideoDislikesActivityPubUrl, |
175 | getLocalVideoViewerActivityPubUrl, | ||
170 | 176 | ||
171 | getAbuseTargetUrl, | 177 | getAbuseTargetUrl, |
172 | checkUrlsSameHost, | 178 | checkUrlsSameHost, |
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts index c97217669..f02b9cba6 100644 --- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts +++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts | |||
@@ -24,6 +24,7 @@ import { | |||
24 | VideoPrivacy, | 24 | VideoPrivacy, |
25 | VideoStreamingPlaylistType | 25 | VideoStreamingPlaylistType |
26 | } from '@shared/models' | 26 | } from '@shared/models' |
27 | import { getDurationFromActivityStream } from '../../activity' | ||
27 | 28 | ||
28 | function getThumbnailFromIcons (videoObject: VideoObject) { | 29 | function getThumbnailFromIcons (videoObject: VideoObject) { |
29 | let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth) | 30 | let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth) |
@@ -170,7 +171,6 @@ function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: Vi | |||
170 | ? VideoPrivacy.PUBLIC | 171 | ? VideoPrivacy.PUBLIC |
171 | : VideoPrivacy.UNLISTED | 172 | : VideoPrivacy.UNLISTED |
172 | 173 | ||
173 | const duration = videoObject.duration.replace(/[^\d]+/, '') | ||
174 | const language = videoObject.language?.identifier | 174 | const language = videoObject.language?.identifier |
175 | 175 | ||
176 | const category = videoObject.category | 176 | const category = videoObject.category |
@@ -200,7 +200,7 @@ function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: Vi | |||
200 | isLive: videoObject.isLiveBroadcast, | 200 | isLive: videoObject.isLiveBroadcast, |
201 | state: videoObject.state, | 201 | state: videoObject.state, |
202 | channelId: videoChannel.id, | 202 | channelId: videoChannel.id, |
203 | duration: parseInt(duration, 10), | 203 | duration: getDurationFromActivityStream(videoObject.duration), |
204 | createdAt: new Date(videoObject.published), | 204 | createdAt: new Date(videoObject.published), |
205 | publishedAt: new Date(videoObject.published), | 205 | publishedAt: new Date(videoObject.published), |
206 | 206 | ||
diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index a9c835fbf..337364ac9 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts | |||
@@ -23,11 +23,11 @@ import { | |||
23 | WEBSERVER | 23 | WEBSERVER |
24 | } from '../initializers/constants' | 24 | } from '../initializers/constants' |
25 | import { AccountModel } from '../models/account/account' | 25 | import { AccountModel } from '../models/account/account' |
26 | import { getActivityStreamDuration } from '../models/video/formatter/video-format-utils' | ||
27 | import { VideoModel } from '../models/video/video' | 26 | import { VideoModel } from '../models/video/video' |
28 | import { VideoChannelModel } from '../models/video/video-channel' | 27 | import { VideoChannelModel } from '../models/video/video-channel' |
29 | import { VideoPlaylistModel } from '../models/video/video-playlist' | 28 | import { VideoPlaylistModel } from '../models/video/video-playlist' |
30 | import { MAccountActor, MChannelActor } from '../types/models' | 29 | import { MAccountActor, MChannelActor } from '../types/models' |
30 | import { getActivityStreamDuration } from './activitypub/activity' | ||
31 | import { getBiggestActorImage } from './actor-image' | 31 | import { getBiggestActorImage } from './actor-image' |
32 | import { ServerConfigManager } from './server-config-manager' | 32 | import { ServerConfigManager } from './server-config-manager' |
33 | 33 | ||
diff --git a/server/lib/job-queue/handlers/video-views-stats.ts b/server/lib/job-queue/handlers/video-views-stats.ts index caf5f6962..689a5a3b4 100644 --- a/server/lib/job-queue/handlers/video-views-stats.ts +++ b/server/lib/job-queue/handlers/video-views-stats.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { VideoViewModel } from '@server/models/view/video-view' | ||
1 | import { isTestInstance } from '../../../helpers/core-utils' | 2 | import { isTestInstance } from '../../../helpers/core-utils' |
2 | import { logger } from '../../../helpers/logger' | 3 | import { logger } from '../../../helpers/logger' |
3 | import { VideoModel } from '../../../models/video/video' | 4 | import { VideoModel } from '../../../models/video/video' |
4 | import { VideoViewModel } from '../../../models/video/video-view' | ||
5 | import { Redis } from '../../redis' | 5 | import { Redis } from '../../redis' |
6 | 6 | ||
7 | async function processVideosViewsStats () { | 7 | async function processVideosViewsStats () { |
diff --git a/server/lib/redis.ts b/server/lib/redis.ts index c4c1fa443..b86aefa0e 100644 --- a/server/lib/redis.ts +++ b/server/lib/redis.ts | |||
@@ -249,6 +249,45 @@ class Redis { | |||
249 | ]) | 249 | ]) |
250 | } | 250 | } |
251 | 251 | ||
252 | /* ************ Video viewers stats ************ */ | ||
253 | |||
254 | getLocalVideoViewer (options: { | ||
255 | key?: string | ||
256 | // Or | ||
257 | ip?: string | ||
258 | videoId?: number | ||
259 | }) { | ||
260 | if (options.key) return this.getObject(options.key) | ||
261 | |||
262 | const { viewerKey } = this.generateLocalVideoViewerKeys(options.ip, options.videoId) | ||
263 | |||
264 | return this.getObject(viewerKey) | ||
265 | } | ||
266 | |||
267 | setLocalVideoViewer (ip: string, videoId: number, object: any) { | ||
268 | const { setKey, viewerKey } = this.generateLocalVideoViewerKeys(ip, videoId) | ||
269 | |||
270 | return Promise.all([ | ||
271 | this.addToSet(setKey, viewerKey), | ||
272 | this.setObject(viewerKey, object) | ||
273 | ]) | ||
274 | } | ||
275 | |||
276 | listLocalVideoViewerKeys () { | ||
277 | const { setKey } = this.generateLocalVideoViewerKeys() | ||
278 | |||
279 | return this.getSet(setKey) | ||
280 | } | ||
281 | |||
282 | deleteLocalVideoViewersKeys (key: string) { | ||
283 | const { setKey } = this.generateLocalVideoViewerKeys() | ||
284 | |||
285 | return Promise.all([ | ||
286 | this.deleteFromSet(setKey, key), | ||
287 | this.deleteKey(key) | ||
288 | ]) | ||
289 | } | ||
290 | |||
252 | /* ************ Resumable uploads final responses ************ */ | 291 | /* ************ Resumable uploads final responses ************ */ |
253 | 292 | ||
254 | setUploadSession (uploadId: string, response?: { video: { id: number, shortUUID: string, uuid: string } }) { | 293 | setUploadSession (uploadId: string, response?: { video: { id: number, shortUUID: string, uuid: string } }) { |
@@ -290,10 +329,18 @@ class Redis { | |||
290 | 329 | ||
291 | /* ************ Keys generation ************ */ | 330 | /* ************ Keys generation ************ */ |
292 | 331 | ||
293 | private generateLocalVideoViewsKeys (videoId?: Number) { | 332 | private generateLocalVideoViewsKeys (videoId: number): { setKey: string, videoKey: string } |
333 | private generateLocalVideoViewsKeys (): { setKey: string } | ||
334 | private generateLocalVideoViewsKeys (videoId?: number) { | ||
294 | return { setKey: `local-video-views-buffer`, videoKey: `local-video-views-buffer-${videoId}` } | 335 | return { setKey: `local-video-views-buffer`, videoKey: `local-video-views-buffer-${videoId}` } |
295 | } | 336 | } |
296 | 337 | ||
338 | private generateLocalVideoViewerKeys (ip: string, videoId: number): { setKey: string, viewerKey: string } | ||
339 | private generateLocalVideoViewerKeys (): { setKey: string } | ||
340 | private generateLocalVideoViewerKeys (ip?: string, videoId?: number) { | ||
341 | return { setKey: `local-video-viewer-stats-keys`, viewerKey: `local-video-viewer-stats-${ip}-${videoId}` } | ||
342 | } | ||
343 | |||
297 | private generateVideoViewStatsKeys (options: { videoId?: number, hour?: number }) { | 344 | private generateVideoViewStatsKeys (options: { videoId?: number, hour?: number }) { |
298 | const hour = exists(options.hour) | 345 | const hour = exists(options.hour) |
299 | ? options.hour | 346 | ? options.hour |
@@ -352,8 +399,23 @@ class Redis { | |||
352 | return this.client.del(this.prefix + key) | 399 | return this.client.del(this.prefix + key) |
353 | } | 400 | } |
354 | 401 | ||
355 | private async setValue (key: string, value: string, expirationMilliseconds: number) { | 402 | private async getObject (key: string) { |
356 | const result = await this.client.set(this.prefix + key, value, { PX: expirationMilliseconds }) | 403 | const value = await this.getValue(key) |
404 | if (!value) return null | ||
405 | |||
406 | return JSON.parse(value) | ||
407 | } | ||
408 | |||
409 | private setObject (key: string, value: { [ id: string ]: number | string }) { | ||
410 | return this.setValue(key, JSON.stringify(value)) | ||
411 | } | ||
412 | |||
413 | private async setValue (key: string, value: string, expirationMilliseconds?: number) { | ||
414 | const options = expirationMilliseconds | ||
415 | ? { PX: expirationMilliseconds } | ||
416 | : {} | ||
417 | |||
418 | const result = await this.client.set(this.prefix + key, value, options) | ||
357 | 419 | ||
358 | if (result !== 'OK') throw new Error('Redis set result is not OK.') | 420 | if (result !== 'OK') throw new Error('Redis set result is not OK.') |
359 | } | 421 | } |
diff --git a/server/lib/schedulers/geo-ip-update-scheduler.ts b/server/lib/schedulers/geo-ip-update-scheduler.ts new file mode 100644 index 000000000..9dda6d76c --- /dev/null +++ b/server/lib/schedulers/geo-ip-update-scheduler.ts | |||
@@ -0,0 +1,22 @@ | |||
1 | import { GeoIP } from '@server/helpers/geo-ip' | ||
2 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | ||
3 | import { AbstractScheduler } from './abstract-scheduler' | ||
4 | |||
5 | export class GeoIPUpdateScheduler extends AbstractScheduler { | ||
6 | |||
7 | private static instance: AbstractScheduler | ||
8 | |||
9 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.YOUTUBE_DL_UPDATE | ||
10 | |||
11 | private constructor () { | ||
12 | super() | ||
13 | } | ||
14 | |||
15 | protected internalExecute () { | ||
16 | return GeoIP.Instance.updateDatabase() | ||
17 | } | ||
18 | |||
19 | static get Instance () { | ||
20 | return this.instance || (this.instance = new this()) | ||
21 | } | ||
22 | } | ||
diff --git a/server/lib/schedulers/remove-old-views-scheduler.ts b/server/lib/schedulers/remove-old-views-scheduler.ts index 64bef97fe..8bc53a045 100644 --- a/server/lib/schedulers/remove-old-views-scheduler.ts +++ b/server/lib/schedulers/remove-old-views-scheduler.ts | |||
@@ -1,8 +1,8 @@ | |||
1 | import { VideoViewModel } from '@server/models/view/video-view' | ||
1 | import { logger } from '../../helpers/logger' | 2 | import { logger } from '../../helpers/logger' |
2 | import { AbstractScheduler } from './abstract-scheduler' | ||
3 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | ||
4 | import { CONFIG } from '../../initializers/config' | 3 | import { CONFIG } from '../../initializers/config' |
5 | import { VideoViewModel } from '../../models/video/video-view' | 4 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' |
5 | import { AbstractScheduler } from './abstract-scheduler' | ||
6 | 6 | ||
7 | export class RemoveOldViewsScheduler extends AbstractScheduler { | 7 | export class RemoveOldViewsScheduler extends AbstractScheduler { |
8 | 8 | ||
diff --git a/server/lib/schedulers/video-views-buffer-scheduler.ts b/server/lib/schedulers/video-views-buffer-scheduler.ts index c0e72c461..937764155 100644 --- a/server/lib/schedulers/video-views-buffer-scheduler.ts +++ b/server/lib/schedulers/video-views-buffer-scheduler.ts | |||
@@ -21,8 +21,6 @@ export class VideoViewsBufferScheduler extends AbstractScheduler { | |||
21 | const videoIds = await Redis.Instance.listLocalVideosViewed() | 21 | const videoIds = await Redis.Instance.listLocalVideosViewed() |
22 | if (videoIds.length === 0) return | 22 | if (videoIds.length === 0) return |
23 | 23 | ||
24 | logger.info('Processing local video views buffer.', { videoIds, ...lTags() }) | ||
25 | |||
26 | for (const videoId of videoIds) { | 24 | for (const videoId of videoIds) { |
27 | try { | 25 | try { |
28 | const views = await Redis.Instance.getLocalVideoViews(videoId) | 26 | const views = await Redis.Instance.getLocalVideoViews(videoId) |
@@ -34,6 +32,8 @@ export class VideoViewsBufferScheduler extends AbstractScheduler { | |||
34 | continue | 32 | continue |
35 | } | 33 | } |
36 | 34 | ||
35 | logger.info('Processing local video %s views buffer.', video.uuid, lTags(video.uuid)) | ||
36 | |||
37 | // If this is a remote video, the origin instance will send us an update | 37 | // If this is a remote video, the origin instance will send us an update |
38 | await VideoModel.incrementViews(videoId, views) | 38 | await VideoModel.incrementViews(videoId, views) |
39 | 39 | ||
diff --git a/server/lib/video-views.ts b/server/lib/video-views.ts deleted file mode 100644 index c024eb93c..000000000 --- a/server/lib/video-views.ts +++ /dev/null | |||
@@ -1,131 +0,0 @@ | |||
1 | import { isTestInstance } from '@server/helpers/core-utils' | ||
2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
3 | import { VIEW_LIFETIME } from '@server/initializers/constants' | ||
4 | import { VideoModel } from '@server/models/video/video' | ||
5 | import { MVideo } from '@server/types/models' | ||
6 | import { PeerTubeSocket } from './peertube-socket' | ||
7 | import { Redis } from './redis' | ||
8 | |||
9 | const lTags = loggerTagsFactory('views') | ||
10 | |||
11 | export class VideoViews { | ||
12 | |||
13 | // Values are Date().getTime() | ||
14 | private readonly viewersPerVideo = new Map<number, number[]>() | ||
15 | |||
16 | private static instance: VideoViews | ||
17 | |||
18 | private constructor () { | ||
19 | } | ||
20 | |||
21 | init () { | ||
22 | setInterval(() => this.cleanViewers(), VIEW_LIFETIME.VIEWER) | ||
23 | } | ||
24 | |||
25 | async processView (options: { | ||
26 | video: MVideo | ||
27 | ip: string | null | ||
28 | viewerExpires?: Date | ||
29 | }) { | ||
30 | const { video, ip, viewerExpires } = options | ||
31 | |||
32 | logger.debug('Processing view for %s and ip %s.', video.url, ip, lTags()) | ||
33 | |||
34 | let success = await this.addView(video, ip) | ||
35 | |||
36 | if (video.isLive) { | ||
37 | const successViewer = await this.addViewer(video, ip, viewerExpires) | ||
38 | success ||= successViewer | ||
39 | } | ||
40 | |||
41 | return success | ||
42 | } | ||
43 | |||
44 | getViewers (video: MVideo) { | ||
45 | const viewers = this.viewersPerVideo.get(video.id) | ||
46 | if (!viewers) return 0 | ||
47 | |||
48 | return viewers.length | ||
49 | } | ||
50 | |||
51 | buildViewerExpireTime () { | ||
52 | return new Date().getTime() + VIEW_LIFETIME.VIEWER | ||
53 | } | ||
54 | |||
55 | private async addView (video: MVideo, ip: string | null) { | ||
56 | const promises: Promise<any>[] = [] | ||
57 | |||
58 | if (ip !== null) { | ||
59 | const viewExists = await Redis.Instance.doesVideoIPViewExist(ip, video.uuid) | ||
60 | if (viewExists) return false | ||
61 | |||
62 | promises.push(Redis.Instance.setIPVideoView(ip, video.uuid)) | ||
63 | } | ||
64 | |||
65 | if (video.isOwned()) { | ||
66 | promises.push(Redis.Instance.addLocalVideoView(video.id)) | ||
67 | } | ||
68 | |||
69 | promises.push(Redis.Instance.addVideoViewStats(video.id)) | ||
70 | |||
71 | await Promise.all(promises) | ||
72 | |||
73 | return true | ||
74 | } | ||
75 | |||
76 | private async addViewer (video: MVideo, ip: string | null, viewerExpires?: Date) { | ||
77 | if (ip !== null) { | ||
78 | const viewExists = await Redis.Instance.doesVideoIPViewerExist(ip, video.uuid) | ||
79 | if (viewExists) return false | ||
80 | |||
81 | await Redis.Instance.setIPVideoViewer(ip, video.uuid) | ||
82 | } | ||
83 | |||
84 | let watchers = this.viewersPerVideo.get(video.id) | ||
85 | |||
86 | if (!watchers) { | ||
87 | watchers = [] | ||
88 | this.viewersPerVideo.set(video.id, watchers) | ||
89 | } | ||
90 | |||
91 | const expiration = viewerExpires | ||
92 | ? viewerExpires.getTime() | ||
93 | : this.buildViewerExpireTime() | ||
94 | |||
95 | watchers.push(expiration) | ||
96 | await this.notifyClients(video.id, watchers.length) | ||
97 | |||
98 | return true | ||
99 | } | ||
100 | |||
101 | private async cleanViewers () { | ||
102 | if (!isTestInstance()) logger.info('Cleaning video viewers.', lTags()) | ||
103 | |||
104 | for (const videoId of this.viewersPerVideo.keys()) { | ||
105 | const notBefore = new Date().getTime() | ||
106 | |||
107 | const viewers = this.viewersPerVideo.get(videoId) | ||
108 | |||
109 | // Only keep not expired viewers | ||
110 | const newViewers = viewers.filter(w => w > notBefore) | ||
111 | |||
112 | if (newViewers.length === 0) this.viewersPerVideo.delete(videoId) | ||
113 | else this.viewersPerVideo.set(videoId, newViewers) | ||
114 | |||
115 | await this.notifyClients(videoId, newViewers.length) | ||
116 | } | ||
117 | } | ||
118 | |||
119 | private async notifyClients (videoId: string | number, viewersLength: number) { | ||
120 | const video = await VideoModel.loadImmutableAttributes(videoId) | ||
121 | if (!video) return | ||
122 | |||
123 | PeerTubeSocket.Instance.sendVideoViewsUpdate(video, viewersLength) | ||
124 | |||
125 | logger.debug('Live video views update for %s is %d.', video.url, viewersLength, lTags()) | ||
126 | } | ||
127 | |||
128 | static get Instance () { | ||
129 | return this.instance || (this.instance = new this()) | ||
130 | } | ||
131 | } | ||
diff --git a/server/lib/views/shared/index.ts b/server/lib/views/shared/index.ts new file mode 100644 index 000000000..dd510f4e2 --- /dev/null +++ b/server/lib/views/shared/index.ts | |||
@@ -0,0 +1,2 @@ | |||
1 | export * from './video-viewers' | ||
2 | export * from './video-views' | ||
diff --git a/server/lib/views/shared/video-viewers.ts b/server/lib/views/shared/video-viewers.ts new file mode 100644 index 000000000..5c26f8982 --- /dev/null +++ b/server/lib/views/shared/video-viewers.ts | |||
@@ -0,0 +1,276 @@ | |||
1 | import { Transaction } from 'sequelize/types' | ||
2 | import { isTestInstance } from '@server/helpers/core-utils' | ||
3 | import { GeoIP } from '@server/helpers/geo-ip' | ||
4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
5 | import { MAX_LOCAL_VIEWER_WATCH_SECTIONS, VIEW_LIFETIME } from '@server/initializers/constants' | ||
6 | import { sequelizeTypescript } from '@server/initializers/database' | ||
7 | import { sendCreateWatchAction } from '@server/lib/activitypub/send' | ||
8 | import { getLocalVideoViewerActivityPubUrl } from '@server/lib/activitypub/url' | ||
9 | import { PeerTubeSocket } from '@server/lib/peertube-socket' | ||
10 | import { Redis } from '@server/lib/redis' | ||
11 | import { VideoModel } from '@server/models/video/video' | ||
12 | import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' | ||
13 | import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' | ||
14 | import { MVideo } from '@server/types/models' | ||
15 | import { VideoViewEvent } from '@shared/models' | ||
16 | |||
17 | const lTags = loggerTagsFactory('views') | ||
18 | |||
19 | type LocalViewerStats = { | ||
20 | firstUpdated: number // Date.getTime() | ||
21 | lastUpdated: number // Date.getTime() | ||
22 | |||
23 | watchSections: { | ||
24 | start: number | ||
25 | end: number | ||
26 | }[] | ||
27 | |||
28 | watchTime: number | ||
29 | |||
30 | country: string | ||
31 | |||
32 | videoId: number | ||
33 | } | ||
34 | |||
35 | export class VideoViewers { | ||
36 | |||
37 | // Values are Date().getTime() | ||
38 | private readonly viewersPerVideo = new Map<number, number[]>() | ||
39 | |||
40 | private processingViewerCounters = false | ||
41 | private processingViewerStats = false | ||
42 | |||
43 | constructor () { | ||
44 | setInterval(() => this.cleanViewerCounters(), VIEW_LIFETIME.VIEWER) | ||
45 | |||
46 | setInterval(() => this.processViewerStats(), VIEW_LIFETIME.VIEWER_STATS) | ||
47 | } | ||
48 | |||
49 | // --------------------------------------------------------------------------- | ||
50 | |||
51 | getViewers (video: MVideo) { | ||
52 | const viewers = this.viewersPerVideo.get(video.id) | ||
53 | if (!viewers) return 0 | ||
54 | |||
55 | return viewers.length | ||
56 | } | ||
57 | |||
58 | buildViewerExpireTime () { | ||
59 | return new Date().getTime() + VIEW_LIFETIME.VIEWER | ||
60 | } | ||
61 | |||
62 | async getWatchTime (videoId: number, ip: string) { | ||
63 | const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId }) | ||
64 | |||
65 | return stats?.watchTime || 0 | ||
66 | } | ||
67 | |||
68 | async addLocalViewer (options: { | ||
69 | video: MVideo | ||
70 | currentTime: number | ||
71 | ip: string | ||
72 | viewEvent?: VideoViewEvent | ||
73 | }) { | ||
74 | const { video, ip, viewEvent, currentTime } = options | ||
75 | |||
76 | logger.debug('Adding local viewer to video %s.', video.uuid, { currentTime, viewEvent, ...lTags(video.uuid) }) | ||
77 | |||
78 | await this.updateLocalViewerStats({ video, viewEvent, currentTime, ip }) | ||
79 | |||
80 | const viewExists = await Redis.Instance.doesVideoIPViewerExist(ip, video.uuid) | ||
81 | if (viewExists) return false | ||
82 | |||
83 | await Redis.Instance.setIPVideoViewer(ip, video.uuid) | ||
84 | |||
85 | return this.addViewerToVideo({ video }) | ||
86 | } | ||
87 | |||
88 | async addRemoteViewer (options: { | ||
89 | video: MVideo | ||
90 | viewerExpires: Date | ||
91 | }) { | ||
92 | const { video, viewerExpires } = options | ||
93 | |||
94 | logger.debug('Adding remote viewer to video %s.', video.uuid, { ...lTags(video.uuid) }) | ||
95 | |||
96 | return this.addViewerToVideo({ video, viewerExpires }) | ||
97 | } | ||
98 | |||
99 | private async addViewerToVideo (options: { | ||
100 | video: MVideo | ||
101 | viewerExpires?: Date | ||
102 | }) { | ||
103 | const { video, viewerExpires } = options | ||
104 | |||
105 | let watchers = this.viewersPerVideo.get(video.id) | ||
106 | |||
107 | if (!watchers) { | ||
108 | watchers = [] | ||
109 | this.viewersPerVideo.set(video.id, watchers) | ||
110 | } | ||
111 | |||
112 | const expiration = viewerExpires | ||
113 | ? viewerExpires.getTime() | ||
114 | : this.buildViewerExpireTime() | ||
115 | |||
116 | watchers.push(expiration) | ||
117 | await this.notifyClients(video.id, watchers.length) | ||
118 | |||
119 | return true | ||
120 | } | ||
121 | |||
122 | private async updateLocalViewerStats (options: { | ||
123 | video: MVideo | ||
124 | ip: string | ||
125 | currentTime: number | ||
126 | viewEvent?: VideoViewEvent | ||
127 | }) { | ||
128 | const { video, ip, viewEvent, currentTime } = options | ||
129 | const nowMs = new Date().getTime() | ||
130 | |||
131 | let stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId: video.id }) | ||
132 | |||
133 | if (stats && stats.watchSections.length >= MAX_LOCAL_VIEWER_WATCH_SECTIONS) { | ||
134 | logger.warn('Too much watch section to store for a viewer, skipping this one', { currentTime, viewEvent, ...lTags(video.uuid) }) | ||
135 | return | ||
136 | } | ||
137 | |||
138 | if (!stats) { | ||
139 | const country = await GeoIP.Instance.safeCountryISOLookup(ip) | ||
140 | |||
141 | stats = { | ||
142 | firstUpdated: nowMs, | ||
143 | lastUpdated: nowMs, | ||
144 | |||
145 | watchSections: [], | ||
146 | |||
147 | watchTime: 0, | ||
148 | |||
149 | country, | ||
150 | videoId: video.id | ||
151 | } | ||
152 | } | ||
153 | |||
154 | stats.lastUpdated = nowMs | ||
155 | |||
156 | if (viewEvent === 'seek' || stats.watchSections.length === 0) { | ||
157 | stats.watchSections.push({ | ||
158 | start: currentTime, | ||
159 | end: currentTime | ||
160 | }) | ||
161 | } else { | ||
162 | const lastSection = stats.watchSections[stats.watchSections.length - 1] | ||
163 | lastSection.end = currentTime | ||
164 | } | ||
165 | |||
166 | stats.watchTime = this.buildWatchTimeFromSections(stats.watchSections) | ||
167 | |||
168 | logger.debug('Set local video viewer stats for video %s.', video.uuid, { stats, ...lTags(video.uuid) }) | ||
169 | |||
170 | await Redis.Instance.setLocalVideoViewer(ip, video.id, stats) | ||
171 | } | ||
172 | |||
173 | private async cleanViewerCounters () { | ||
174 | if (this.processingViewerCounters) return | ||
175 | this.processingViewerCounters = true | ||
176 | |||
177 | if (!isTestInstance()) logger.info('Cleaning video viewers.', lTags()) | ||
178 | |||
179 | try { | ||
180 | for (const videoId of this.viewersPerVideo.keys()) { | ||
181 | const notBefore = new Date().getTime() | ||
182 | |||
183 | const viewers = this.viewersPerVideo.get(videoId) | ||
184 | |||
185 | // Only keep not expired viewers | ||
186 | const newViewers = viewers.filter(w => w > notBefore) | ||
187 | |||
188 | if (newViewers.length === 0) this.viewersPerVideo.delete(videoId) | ||
189 | else this.viewersPerVideo.set(videoId, newViewers) | ||
190 | |||
191 | await this.notifyClients(videoId, newViewers.length) | ||
192 | } | ||
193 | } catch (err) { | ||
194 | logger.error('Error in video clean viewers scheduler.', { err, ...lTags() }) | ||
195 | } | ||
196 | |||
197 | this.processingViewerCounters = false | ||
198 | } | ||
199 | |||
200 | private async notifyClients (videoId: string | number, viewersLength: number) { | ||
201 | const video = await VideoModel.loadImmutableAttributes(videoId) | ||
202 | if (!video) return | ||
203 | |||
204 | PeerTubeSocket.Instance.sendVideoViewsUpdate(video, viewersLength) | ||
205 | |||
206 | logger.debug('Video viewers update for %s is %d.', video.url, viewersLength, lTags()) | ||
207 | } | ||
208 | |||
209 | async processViewerStats () { | ||
210 | if (this.processingViewerStats) return | ||
211 | this.processingViewerStats = true | ||
212 | |||
213 | if (!isTestInstance()) logger.info('Processing viewers.', lTags()) | ||
214 | |||
215 | const now = new Date().getTime() | ||
216 | |||
217 | try { | ||
218 | const allKeys = await Redis.Instance.listLocalVideoViewerKeys() | ||
219 | |||
220 | for (const key of allKeys) { | ||
221 | const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ key }) | ||
222 | |||
223 | if (stats.lastUpdated > now - VIEW_LIFETIME.VIEWER_STATS) { | ||
224 | continue | ||
225 | } | ||
226 | |||
227 | try { | ||
228 | await sequelizeTypescript.transaction(async t => { | ||
229 | const video = await VideoModel.load(stats.videoId, t) | ||
230 | |||
231 | const statsModel = await this.saveViewerStats(video, stats, t) | ||
232 | |||
233 | if (video.remote) { | ||
234 | await sendCreateWatchAction(statsModel, t) | ||
235 | } | ||
236 | }) | ||
237 | |||
238 | await Redis.Instance.deleteLocalVideoViewersKeys(key) | ||
239 | } catch (err) { | ||
240 | logger.error('Cannot process viewer stats for Redis key %s.', key, { err, ...lTags() }) | ||
241 | } | ||
242 | } | ||
243 | } catch (err) { | ||
244 | logger.error('Error in video save viewers stats scheduler.', { err, ...lTags() }) | ||
245 | } | ||
246 | |||
247 | this.processingViewerStats = false | ||
248 | } | ||
249 | |||
250 | private async saveViewerStats (video: MVideo, stats: LocalViewerStats, transaction: Transaction) { | ||
251 | const statsModel = new LocalVideoViewerModel({ | ||
252 | startDate: new Date(stats.firstUpdated), | ||
253 | endDate: new Date(stats.lastUpdated), | ||
254 | watchTime: stats.watchTime, | ||
255 | country: stats.country, | ||
256 | videoId: video.id | ||
257 | }) | ||
258 | |||
259 | statsModel.url = getLocalVideoViewerActivityPubUrl(statsModel) | ||
260 | statsModel.Video = video as VideoModel | ||
261 | |||
262 | await statsModel.save({ transaction }) | ||
263 | |||
264 | statsModel.WatchSections = await LocalVideoViewerWatchSectionModel.bulkCreateSections({ | ||
265 | localVideoViewerId: statsModel.id, | ||
266 | watchSections: stats.watchSections, | ||
267 | transaction | ||
268 | }) | ||
269 | |||
270 | return statsModel | ||
271 | } | ||
272 | |||
273 | private buildWatchTimeFromSections (sections: { start: number, end: number }[]) { | ||
274 | return sections.reduce((p, current) => p + (current.end - current.start), 0) | ||
275 | } | ||
276 | } | ||
diff --git a/server/lib/views/shared/video-views.ts b/server/lib/views/shared/video-views.ts new file mode 100644 index 000000000..19250f993 --- /dev/null +++ b/server/lib/views/shared/video-views.ts | |||
@@ -0,0 +1,60 @@ | |||
1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
2 | import { MVideo } from '@server/types/models' | ||
3 | import { Redis } from '../../redis' | ||
4 | |||
5 | const lTags = loggerTagsFactory('views') | ||
6 | |||
7 | export class VideoViews { | ||
8 | |||
9 | async addLocalView (options: { | ||
10 | video: MVideo | ||
11 | ip: string | ||
12 | watchTime: number | ||
13 | }) { | ||
14 | const { video, ip, watchTime } = options | ||
15 | |||
16 | logger.debug('Adding local view to video %s.', video.uuid, { watchTime, ...lTags(video.uuid) }) | ||
17 | |||
18 | if (!this.hasEnoughWatchTime(video, watchTime)) return false | ||
19 | |||
20 | const viewExists = await Redis.Instance.doesVideoIPViewExist(ip, video.uuid) | ||
21 | if (viewExists) return false | ||
22 | |||
23 | await Redis.Instance.setIPVideoView(ip, video.uuid) | ||
24 | |||
25 | await this.addView(video) | ||
26 | |||
27 | return true | ||
28 | } | ||
29 | |||
30 | async addRemoteView (options: { | ||
31 | video: MVideo | ||
32 | }) { | ||
33 | const { video } = options | ||
34 | |||
35 | logger.debug('Adding remote view to video %s.', video.uuid, { ...lTags(video.uuid) }) | ||
36 | |||
37 | await this.addView(video) | ||
38 | |||
39 | return true | ||
40 | } | ||
41 | |||
42 | private async addView (video: MVideo) { | ||
43 | const promises: Promise<any>[] = [] | ||
44 | |||
45 | if (video.isOwned()) { | ||
46 | promises.push(Redis.Instance.addLocalVideoView(video.id)) | ||
47 | } | ||
48 | |||
49 | promises.push(Redis.Instance.addVideoViewStats(video.id)) | ||
50 | |||
51 | await Promise.all(promises) | ||
52 | } | ||
53 | |||
54 | private hasEnoughWatchTime (video: MVideo, watchTime: number) { | ||
55 | if (video.isLive || video.duration >= 30) return watchTime >= 30 | ||
56 | |||
57 | // Check more than 50% of the video is watched | ||
58 | return video.duration / watchTime < 2 | ||
59 | } | ||
60 | } | ||
diff --git a/server/lib/views/video-views-manager.ts b/server/lib/views/video-views-manager.ts new file mode 100644 index 000000000..e07af1ca9 --- /dev/null +++ b/server/lib/views/video-views-manager.ts | |||
@@ -0,0 +1,70 @@ | |||
1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
2 | import { MVideo } from '@server/types/models' | ||
3 | import { VideoViewEvent } from '@shared/models' | ||
4 | import { VideoViewers, VideoViews } from './shared' | ||
5 | |||
6 | const lTags = loggerTagsFactory('views') | ||
7 | |||
8 | export class VideoViewsManager { | ||
9 | |||
10 | private static instance: VideoViewsManager | ||
11 | |||
12 | private videoViewers: VideoViewers | ||
13 | private videoViews: VideoViews | ||
14 | |||
15 | private constructor () { | ||
16 | } | ||
17 | |||
18 | init () { | ||
19 | this.videoViewers = new VideoViewers() | ||
20 | this.videoViews = new VideoViews() | ||
21 | } | ||
22 | |||
23 | async processLocalView (options: { | ||
24 | video: MVideo | ||
25 | currentTime: number | ||
26 | ip: string | null | ||
27 | viewEvent?: VideoViewEvent | ||
28 | }) { | ||
29 | const { video, ip, viewEvent, currentTime } = options | ||
30 | |||
31 | logger.debug('Processing local view for %s and ip %s.', video.url, ip, lTags()) | ||
32 | |||
33 | const successViewer = await this.videoViewers.addLocalViewer({ video, ip, viewEvent, currentTime }) | ||
34 | |||
35 | // Do it after added local viewer to fetch updated information | ||
36 | const watchTime = await this.videoViewers.getWatchTime(video.id, ip) | ||
37 | |||
38 | const successView = await this.videoViews.addLocalView({ video, watchTime, ip }) | ||
39 | |||
40 | return { successView, successViewer } | ||
41 | } | ||
42 | |||
43 | async processRemoteView (options: { | ||
44 | video: MVideo | ||
45 | viewerExpires?: Date | ||
46 | }) { | ||
47 | const { video, viewerExpires } = options | ||
48 | |||
49 | logger.debug('Processing remote view for %s.', video.url, { viewerExpires, ...lTags() }) | ||
50 | |||
51 | if (viewerExpires) await this.videoViewers.addRemoteViewer({ video, viewerExpires }) | ||
52 | else await this.videoViews.addRemoteView({ video }) | ||
53 | } | ||
54 | |||
55 | getViewers (video: MVideo) { | ||
56 | return this.videoViewers.getViewers(video) | ||
57 | } | ||
58 | |||
59 | buildViewerExpireTime () { | ||
60 | return this.videoViewers.buildViewerExpireTime() | ||
61 | } | ||
62 | |||
63 | processViewers () { | ||
64 | return this.videoViewers.processViewerStats() | ||
65 | } | ||
66 | |||
67 | static get Instance () { | ||
68 | return this.instance || (this.instance = new this()) | ||
69 | } | ||
70 | } | ||
diff --git a/server/middlewares/cache/shared/api-cache.ts b/server/middlewares/cache/shared/api-cache.ts index 86c5095b5..abc919339 100644 --- a/server/middlewares/cache/shared/api-cache.ts +++ b/server/middlewares/cache/shared/api-cache.ts | |||
@@ -6,8 +6,8 @@ import { OutgoingHttpHeaders } from 'http' | |||
6 | import { isTestInstance, parseDurationToMs } from '@server/helpers/core-utils' | 6 | import { isTestInstance, parseDurationToMs } from '@server/helpers/core-utils' |
7 | import { logger } from '@server/helpers/logger' | 7 | import { logger } from '@server/helpers/logger' |
8 | import { Redis } from '@server/lib/redis' | 8 | import { Redis } from '@server/lib/redis' |
9 | import { HttpStatusCode } from '@shared/models' | ||
10 | import { asyncMiddleware } from '@server/middlewares' | 9 | import { asyncMiddleware } from '@server/middlewares' |
10 | import { HttpStatusCode } from '@shared/models' | ||
11 | 11 | ||
12 | export interface APICacheOptions { | 12 | export interface APICacheOptions { |
13 | headerBlacklist?: string[] | 13 | headerBlacklist?: string[] |
@@ -152,7 +152,7 @@ export class ApiCache { | |||
152 | end: res.end, | 152 | end: res.end, |
153 | cacheable: true, | 153 | cacheable: true, |
154 | content: undefined, | 154 | content: undefined, |
155 | headers: {} | 155 | headers: undefined |
156 | } | 156 | } |
157 | 157 | ||
158 | // Patch express | 158 | // Patch express |
diff --git a/server/middlewares/validators/express.ts b/server/middlewares/validators/express.ts new file mode 100644 index 000000000..718aec55b --- /dev/null +++ b/server/middlewares/validators/express.ts | |||
@@ -0,0 +1,15 @@ | |||
1 | import * as express from 'express' | ||
2 | |||
3 | const methodsValidator = (methods: string[]) => { | ||
4 | return (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
5 | if (methods.includes(req.method) !== true) { | ||
6 | return res.sendStatus(405) | ||
7 | } | ||
8 | |||
9 | return next() | ||
10 | } | ||
11 | } | ||
12 | |||
13 | export { | ||
14 | methodsValidator | ||
15 | } | ||
diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts index 94a3c2dea..b0ad04819 100644 --- a/server/middlewares/validators/index.ts +++ b/server/middlewares/validators/index.ts | |||
@@ -1,17 +1,26 @@ | |||
1 | export * from './activitypub' | ||
2 | export * from './videos' | ||
1 | export * from './abuse' | 3 | export * from './abuse' |
2 | export * from './account' | 4 | export * from './account' |
3 | export * from './actor-image' | 5 | export * from './actor-image' |
4 | export * from './blocklist' | 6 | export * from './blocklist' |
7 | export * from './bulk' | ||
8 | export * from './config' | ||
9 | export * from './express' | ||
10 | export * from './feeds' | ||
11 | export * from './follows' | ||
12 | export * from './jobs' | ||
13 | export * from './logs' | ||
5 | export * from './oembed' | 14 | export * from './oembed' |
6 | export * from './activitypub' | ||
7 | export * from './pagination' | 15 | export * from './pagination' |
8 | export * from './follows' | 16 | export * from './plugins' |
9 | export * from './feeds' | 17 | export * from './redundancy' |
10 | export * from './sort' | ||
11 | export * from './users' | ||
12 | export * from './user-subscriptions' | ||
13 | export * from './videos' | ||
14 | export * from './search' | 18 | export * from './search' |
15 | export * from './server' | 19 | export * from './server' |
20 | export * from './sort' | ||
21 | export * from './themes' | ||
16 | export * from './user-history' | 22 | export * from './user-history' |
23 | export * from './user-notifications' | ||
24 | export * from './user-subscriptions' | ||
25 | export * from './users' | ||
17 | export * from './webfinger' | 26 | export * from './webfinger' |
diff --git a/server/middlewares/validators/videos/index.ts b/server/middlewares/validators/videos/index.ts index c7dea4b3d..bd2590bc5 100644 --- a/server/middlewares/validators/videos/index.ts +++ b/server/middlewares/validators/videos/index.ts | |||
@@ -6,9 +6,10 @@ export * from './video-files' | |||
6 | export * from './video-imports' | 6 | export * from './video-imports' |
7 | export * from './video-live' | 7 | export * from './video-live' |
8 | export * from './video-ownership-changes' | 8 | export * from './video-ownership-changes' |
9 | export * from './video-watch' | 9 | export * from './video-view' |
10 | export * from './video-rates' | 10 | export * from './video-rates' |
11 | export * from './video-shares' | 11 | export * from './video-shares' |
12 | export * from './video-stats' | ||
12 | export * from './video-studio' | 13 | export * from './video-studio' |
13 | export * from './video-transcoding' | 14 | export * from './video-transcoding' |
14 | export * from './videos' | 15 | export * from './videos' |
diff --git a/server/middlewares/validators/videos/video-stats.ts b/server/middlewares/validators/videos/video-stats.ts new file mode 100644 index 000000000..358b6b473 --- /dev/null +++ b/server/middlewares/validators/videos/video-stats.ts | |||
@@ -0,0 +1,73 @@ | |||
1 | import express from 'express' | ||
2 | import { param } from 'express-validator' | ||
3 | import { isValidStatTimeserieMetric } from '@server/helpers/custom-validators/video-stats' | ||
4 | import { HttpStatusCode, UserRight } from '@shared/models' | ||
5 | import { logger } from '../../../helpers/logger' | ||
6 | import { areValidationErrors, checkUserCanManageVideo, doesVideoExist, isValidVideoIdParam } from '../shared' | ||
7 | |||
8 | const videoOverallStatsValidator = [ | ||
9 | isValidVideoIdParam('videoId'), | ||
10 | |||
11 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
12 | logger.debug('Checking videoOverallStatsValidator parameters', { parameters: req.body }) | ||
13 | |||
14 | if (areValidationErrors(req, res)) return | ||
15 | if (!await commonStatsCheck(req, res)) return | ||
16 | |||
17 | return next() | ||
18 | } | ||
19 | ] | ||
20 | |||
21 | const videoRetentionStatsValidator = [ | ||
22 | isValidVideoIdParam('videoId'), | ||
23 | |||
24 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
25 | logger.debug('Checking videoRetentionStatsValidator parameters', { parameters: req.body }) | ||
26 | |||
27 | if (areValidationErrors(req, res)) return | ||
28 | if (!await commonStatsCheck(req, res)) return | ||
29 | |||
30 | if (res.locals.videoAll.isLive) { | ||
31 | return res.fail({ | ||
32 | status: HttpStatusCode.BAD_REQUEST_400, | ||
33 | message: 'Cannot get retention stats of live video' | ||
34 | }) | ||
35 | } | ||
36 | |||
37 | return next() | ||
38 | } | ||
39 | ] | ||
40 | |||
41 | const videoTimeserieStatsValidator = [ | ||
42 | isValidVideoIdParam('videoId'), | ||
43 | |||
44 | param('metric') | ||
45 | .custom(isValidStatTimeserieMetric) | ||
46 | .withMessage('Should have a valid timeserie metric'), | ||
47 | |||
48 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
49 | logger.debug('Checking videoTimeserieStatsValidator parameters', { parameters: req.body }) | ||
50 | |||
51 | if (areValidationErrors(req, res)) return | ||
52 | if (!await commonStatsCheck(req, res)) return | ||
53 | |||
54 | return next() | ||
55 | } | ||
56 | ] | ||
57 | |||
58 | // --------------------------------------------------------------------------- | ||
59 | |||
60 | export { | ||
61 | videoOverallStatsValidator, | ||
62 | videoTimeserieStatsValidator, | ||
63 | videoRetentionStatsValidator | ||
64 | } | ||
65 | |||
66 | // --------------------------------------------------------------------------- | ||
67 | |||
68 | async function commonStatsCheck (req: express.Request, res: express.Response) { | ||
69 | if (!await doesVideoExist(req.params.videoId, res, 'all')) return false | ||
70 | if (!checkUserCanManageVideo(res.locals.oauth.token.User, res.locals.videoAll, UserRight.SEE_ALL_VIDEOS, res)) return false | ||
71 | |||
72 | return true | ||
73 | } | ||
diff --git a/server/middlewares/validators/videos/video-view.ts b/server/middlewares/validators/videos/video-view.ts new file mode 100644 index 000000000..7a4994e8a --- /dev/null +++ b/server/middlewares/validators/videos/video-view.ts | |||
@@ -0,0 +1,74 @@ | |||
1 | import express from 'express' | ||
2 | import { body, param } from 'express-validator' | ||
3 | import { isVideoTimeValid } from '@server/helpers/custom-validators/video-view' | ||
4 | import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' | ||
5 | import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' | ||
6 | import { exists, isIdValid, isIntOrNull, toIntOrNull } from '../../../helpers/custom-validators/misc' | ||
7 | import { logger } from '../../../helpers/logger' | ||
8 | import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared' | ||
9 | |||
10 | const getVideoLocalViewerValidator = [ | ||
11 | param('localViewerId') | ||
12 | .custom(isIdValid).withMessage('Should have a valid local viewer id'), | ||
13 | |||
14 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
15 | logger.debug('Checking getVideoLocalViewerValidator parameters', { parameters: req.params }) | ||
16 | |||
17 | if (areValidationErrors(req, res)) return | ||
18 | |||
19 | const localViewer = await LocalVideoViewerModel.loadFullById(+req.params.localViewerId) | ||
20 | if (!localViewer) { | ||
21 | return res.fail({ | ||
22 | status: HttpStatusCode.NOT_FOUND_404, | ||
23 | message: 'Local viewer not found' | ||
24 | }) | ||
25 | } | ||
26 | |||
27 | res.locals.localViewerFull = localViewer | ||
28 | |||
29 | return next() | ||
30 | } | ||
31 | ] | ||
32 | |||
33 | const videoViewValidator = [ | ||
34 | isValidVideoIdParam('videoId'), | ||
35 | |||
36 | body('currentTime') | ||
37 | .optional() // TODO: remove optional in a few versions, introduced in 4.2 | ||
38 | .customSanitizer(toIntOrNull) | ||
39 | .custom(isIntOrNull).withMessage('Should have correct current time'), | ||
40 | |||
41 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
42 | logger.debug('Checking videoView parameters', { parameters: req.body }) | ||
43 | |||
44 | if (areValidationErrors(req, res)) return | ||
45 | if (!await doesVideoExist(req.params.videoId, res, 'only-video')) return | ||
46 | |||
47 | const video = res.locals.onlyVideo | ||
48 | const videoDuration = video.isLive | ||
49 | ? undefined | ||
50 | : video.duration | ||
51 | |||
52 | if (!exists(req.body.currentTime)) { // TODO: remove in a few versions, introduced in 4.2 | ||
53 | req.body.currentTime = Math.min(videoDuration ?? 0, 30) | ||
54 | } | ||
55 | |||
56 | const currentTime: number = req.body.currentTime | ||
57 | |||
58 | if (!isVideoTimeValid(currentTime, videoDuration)) { | ||
59 | return res.fail({ | ||
60 | status: HttpStatusCode.BAD_REQUEST_400, | ||
61 | message: 'Current time is invalid' | ||
62 | }) | ||
63 | } | ||
64 | |||
65 | return next() | ||
66 | } | ||
67 | ] | ||
68 | |||
69 | // --------------------------------------------------------------------------- | ||
70 | |||
71 | export { | ||
72 | videoViewValidator, | ||
73 | getVideoLocalViewerValidator | ||
74 | } | ||
diff --git a/server/middlewares/validators/videos/video-watch.ts b/server/middlewares/validators/videos/video-watch.ts deleted file mode 100644 index d83710a64..000000000 --- a/server/middlewares/validators/videos/video-watch.ts +++ /dev/null | |||
@@ -1,38 +0,0 @@ | |||
1 | import express from 'express' | ||
2 | import { body } from 'express-validator' | ||
3 | import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' | ||
4 | import { toIntOrNull } from '../../../helpers/custom-validators/misc' | ||
5 | import { logger } from '../../../helpers/logger' | ||
6 | import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared' | ||
7 | |||
8 | const videoWatchingValidator = [ | ||
9 | isValidVideoIdParam('videoId'), | ||
10 | |||
11 | body('currentTime') | ||
12 | .customSanitizer(toIntOrNull) | ||
13 | .isInt().withMessage('Should have correct current time'), | ||
14 | |||
15 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
16 | logger.debug('Checking videoWatching parameters', { parameters: req.body }) | ||
17 | |||
18 | if (areValidationErrors(req, res)) return | ||
19 | if (!await doesVideoExist(req.params.videoId, res, 'id')) return | ||
20 | |||
21 | const user = res.locals.oauth.token.User | ||
22 | if (user.videosHistoryEnabled === false) { | ||
23 | logger.warn('Cannot set videos to watch by user %d: videos history is disabled.', user.id) | ||
24 | return res.fail({ | ||
25 | status: HttpStatusCode.CONFLICT_409, | ||
26 | message: 'Video history is disabled' | ||
27 | }) | ||
28 | } | ||
29 | |||
30 | return next() | ||
31 | } | ||
32 | ] | ||
33 | |||
34 | // --------------------------------------------------------------------------- | ||
35 | |||
36 | export { | ||
37 | videoWatchingValidator | ||
38 | } | ||
diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts index 611edf0b9..6222107d7 100644 --- a/server/models/video/formatter/video-format-utils.ts +++ b/server/models/video/formatter/video-format-utils.ts | |||
@@ -1,11 +1,19 @@ | |||
1 | import { generateMagnetUri } from '@server/helpers/webtorrent' | 1 | import { generateMagnetUri } from '@server/helpers/webtorrent' |
2 | import { getActivityStreamDuration } from '@server/lib/activitypub/activity' | ||
2 | import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls' | 3 | import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls' |
3 | import { VideoViews } from '@server/lib/video-views' | 4 | import { VideoViewsManager } from '@server/lib/views/video-views-manager' |
4 | import { uuidToShort } from '@shared/extra-utils' | 5 | import { uuidToShort } from '@shared/extra-utils' |
5 | import { VideoFile, VideosCommonQueryAfterSanitize } from '@shared/models' | 6 | import { |
6 | import { ActivityTagObject, ActivityUrlObject, VideoObject } from '../../../../shared/models/activitypub/objects' | 7 | ActivityTagObject, |
7 | import { Video, VideoDetails, VideoInclude } from '../../../../shared/models/videos' | 8 | ActivityUrlObject, |
8 | import { VideoStreamingPlaylist } from '../../../../shared/models/videos/video-streaming-playlist.model' | 9 | Video, |
10 | VideoDetails, | ||
11 | VideoFile, | ||
12 | VideoInclude, | ||
13 | VideoObject, | ||
14 | VideosCommonQueryAfterSanitize, | ||
15 | VideoStreamingPlaylist | ||
16 | } from '@shared/models' | ||
9 | import { isArray } from '../../../helpers/custom-validators/misc' | 17 | import { isArray } from '../../../helpers/custom-validators/misc' |
10 | import { | 18 | import { |
11 | MIMETYPES, | 19 | MIMETYPES, |
@@ -97,7 +105,10 @@ function videoModelToFormattedJSON (video: MVideoFormattable, options: VideoForm | |||
97 | 105 | ||
98 | isLocal: video.isOwned(), | 106 | isLocal: video.isOwned(), |
99 | duration: video.duration, | 107 | duration: video.duration, |
108 | |||
100 | views: video.views, | 109 | views: video.views, |
110 | viewers: VideoViewsManager.Instance.getViewers(video), | ||
111 | |||
101 | likes: video.likes, | 112 | likes: video.likes, |
102 | dislikes: video.dislikes, | 113 | dislikes: video.dislikes, |
103 | thumbnailPath: video.getMiniatureStaticPath(), | 114 | thumbnailPath: video.getMiniatureStaticPath(), |
@@ -121,10 +132,6 @@ function videoModelToFormattedJSON (video: MVideoFormattable, options: VideoForm | |||
121 | pluginData: (video as any).pluginData | 132 | pluginData: (video as any).pluginData |
122 | } | 133 | } |
123 | 134 | ||
124 | if (video.isLive) { | ||
125 | videoObject.viewers = VideoViews.Instance.getViewers(video) | ||
126 | } | ||
127 | |||
128 | const add = options.additionalAttributes | 135 | const add = options.additionalAttributes |
129 | if (add?.state === true) { | 136 | if (add?.state === true) { |
130 | videoObject.state = { | 137 | videoObject.state = { |
@@ -459,11 +466,6 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject { | |||
459 | } | 466 | } |
460 | } | 467 | } |
461 | 468 | ||
462 | function getActivityStreamDuration (duration: number) { | ||
463 | // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration | ||
464 | return 'PT' + duration + 'S' | ||
465 | } | ||
466 | |||
467 | function getCategoryLabel (id: number) { | 469 | function getCategoryLabel (id: number) { |
468 | return VIDEO_CATEGORIES[id] || 'Misc' | 470 | return VIDEO_CATEGORIES[id] || 'Misc' |
469 | } | 471 | } |
@@ -489,7 +491,6 @@ export { | |||
489 | videoModelToFormattedDetailsJSON, | 491 | videoModelToFormattedDetailsJSON, |
490 | videoFilesModelToFormattedJSON, | 492 | videoFilesModelToFormattedJSON, |
491 | videoModelToActivityPubObject, | 493 | videoModelToActivityPubObject, |
492 | getActivityStreamDuration, | ||
493 | 494 | ||
494 | guessAdditionalAttributesFromQuery, | 495 | guessAdditionalAttributesFromQuery, |
495 | 496 | ||
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 8bad2a01e..13d81561a 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -106,6 +106,7 @@ import { setAsUpdated } from '../shared' | |||
106 | import { UserModel } from '../user/user' | 106 | import { UserModel } from '../user/user' |
107 | import { UserVideoHistoryModel } from '../user/user-video-history' | 107 | import { UserVideoHistoryModel } from '../user/user-video-history' |
108 | import { buildTrigramSearchIndex, buildWhereIdOrUUID, getVideoSort, isOutdated, throwIfNotValid } from '../utils' | 108 | import { buildTrigramSearchIndex, buildWhereIdOrUUID, getVideoSort, isOutdated, throwIfNotValid } from '../utils' |
109 | import { VideoViewModel } from '../view/video-view' | ||
109 | import { | 110 | import { |
110 | videoFilesModelToFormattedJSON, | 111 | videoFilesModelToFormattedJSON, |
111 | VideoFormattingJSONOptions, | 112 | VideoFormattingJSONOptions, |
@@ -135,7 +136,6 @@ import { VideoPlaylistElementModel } from './video-playlist-element' | |||
135 | import { VideoShareModel } from './video-share' | 136 | import { VideoShareModel } from './video-share' |
136 | import { VideoStreamingPlaylistModel } from './video-streaming-playlist' | 137 | import { VideoStreamingPlaylistModel } from './video-streaming-playlist' |
137 | import { VideoTagModel } from './video-tag' | 138 | import { VideoTagModel } from './video-tag' |
138 | import { VideoViewModel } from './video-view' | ||
139 | 139 | ||
140 | export enum ScopeNames { | 140 | export enum ScopeNames { |
141 | FOR_API = 'FOR_API', | 141 | FOR_API = 'FOR_API', |
diff --git a/server/models/view/local-video-viewer-watch-section.ts b/server/models/view/local-video-viewer-watch-section.ts new file mode 100644 index 000000000..e29bb7847 --- /dev/null +++ b/server/models/view/local-video-viewer-watch-section.ts | |||
@@ -0,0 +1,63 @@ | |||
1 | import { Transaction } from 'sequelize' | ||
2 | import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, Model, Table } from 'sequelize-typescript' | ||
3 | import { MLocalVideoViewerWatchSection } from '@server/types/models' | ||
4 | import { AttributesOnly } from '@shared/typescript-utils' | ||
5 | import { LocalVideoViewerModel } from './local-video-viewer' | ||
6 | |||
7 | @Table({ | ||
8 | tableName: 'localVideoViewerWatchSection', | ||
9 | updatedAt: false, | ||
10 | indexes: [ | ||
11 | { | ||
12 | fields: [ 'localVideoViewerId' ] | ||
13 | } | ||
14 | ] | ||
15 | }) | ||
16 | export class LocalVideoViewerWatchSectionModel extends Model<Partial<AttributesOnly<LocalVideoViewerWatchSectionModel>>> { | ||
17 | @CreatedAt | ||
18 | createdAt: Date | ||
19 | |||
20 | @AllowNull(false) | ||
21 | @Column | ||
22 | watchStart: number | ||
23 | |||
24 | @AllowNull(false) | ||
25 | @Column | ||
26 | watchEnd: number | ||
27 | |||
28 | @ForeignKey(() => LocalVideoViewerModel) | ||
29 | @Column | ||
30 | localVideoViewerId: number | ||
31 | |||
32 | @BelongsTo(() => LocalVideoViewerModel, { | ||
33 | foreignKey: { | ||
34 | allowNull: false | ||
35 | }, | ||
36 | onDelete: 'CASCADE' | ||
37 | }) | ||
38 | LocalVideoViewer: LocalVideoViewerModel | ||
39 | |||
40 | static async bulkCreateSections (options: { | ||
41 | localVideoViewerId: number | ||
42 | watchSections: { | ||
43 | start: number | ||
44 | end: number | ||
45 | }[] | ||
46 | transaction?: Transaction | ||
47 | }) { | ||
48 | const { localVideoViewerId, watchSections, transaction } = options | ||
49 | const models: MLocalVideoViewerWatchSection[] = [] | ||
50 | |||
51 | for (const section of watchSections) { | ||
52 | const model = await this.create({ | ||
53 | watchStart: section.start, | ||
54 | watchEnd: section.end, | ||
55 | localVideoViewerId | ||
56 | }, { transaction }) | ||
57 | |||
58 | models.push(model) | ||
59 | } | ||
60 | |||
61 | return models | ||
62 | } | ||
63 | } | ||
diff --git a/server/models/view/local-video-viewer.ts b/server/models/view/local-video-viewer.ts new file mode 100644 index 000000000..6f8de53cd --- /dev/null +++ b/server/models/view/local-video-viewer.ts | |||
@@ -0,0 +1,274 @@ | |||
1 | import { QueryTypes } from 'sequelize' | ||
2 | import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, HasMany, IsUUID, Model, Table } from 'sequelize-typescript' | ||
3 | import { STATS_TIMESERIE } from '@server/initializers/constants' | ||
4 | import { getActivityStreamDuration } from '@server/lib/activitypub/activity' | ||
5 | import { MLocalVideoViewer, MLocalVideoViewerWithWatchSections, MVideo } from '@server/types/models' | ||
6 | import { VideoStatsOverall, VideoStatsRetention, VideoStatsTimeserie, VideoStatsTimeserieMetric, WatchActionObject } from '@shared/models' | ||
7 | import { AttributesOnly } from '@shared/typescript-utils' | ||
8 | import { VideoModel } from '../video/video' | ||
9 | import { LocalVideoViewerWatchSectionModel } from './local-video-viewer-watch-section' | ||
10 | |||
11 | @Table({ | ||
12 | tableName: 'localVideoViewer', | ||
13 | updatedAt: false, | ||
14 | indexes: [ | ||
15 | { | ||
16 | fields: [ 'videoId' ] | ||
17 | } | ||
18 | ] | ||
19 | }) | ||
20 | export class LocalVideoViewerModel extends Model<Partial<AttributesOnly<LocalVideoViewerModel>>> { | ||
21 | @CreatedAt | ||
22 | createdAt: Date | ||
23 | |||
24 | @AllowNull(false) | ||
25 | @Column(DataType.DATE) | ||
26 | startDate: Date | ||
27 | |||
28 | @AllowNull(false) | ||
29 | @Column(DataType.DATE) | ||
30 | endDate: Date | ||
31 | |||
32 | @AllowNull(false) | ||
33 | @Column | ||
34 | watchTime: number | ||
35 | |||
36 | @AllowNull(true) | ||
37 | @Column | ||
38 | country: string | ||
39 | |||
40 | @AllowNull(false) | ||
41 | @Default(DataType.UUIDV4) | ||
42 | @IsUUID(4) | ||
43 | @Column(DataType.UUID) | ||
44 | uuid: string | ||
45 | |||
46 | @AllowNull(false) | ||
47 | @Column | ||
48 | url: string | ||
49 | |||
50 | @ForeignKey(() => VideoModel) | ||
51 | @Column | ||
52 | videoId: number | ||
53 | |||
54 | @BelongsTo(() => VideoModel, { | ||
55 | foreignKey: { | ||
56 | allowNull: false | ||
57 | }, | ||
58 | onDelete: 'CASCADE' | ||
59 | }) | ||
60 | Video: VideoModel | ||
61 | |||
62 | @HasMany(() => LocalVideoViewerWatchSectionModel, { | ||
63 | foreignKey: { | ||
64 | allowNull: false | ||
65 | }, | ||
66 | onDelete: 'cascade' | ||
67 | }) | ||
68 | WatchSections: LocalVideoViewerWatchSectionModel[] | ||
69 | |||
70 | static loadByUrl (url: string): Promise<MLocalVideoViewer> { | ||
71 | return this.findOne({ | ||
72 | where: { | ||
73 | url | ||
74 | } | ||
75 | }) | ||
76 | } | ||
77 | |||
78 | static loadFullById (id: number): Promise<MLocalVideoViewerWithWatchSections> { | ||
79 | return this.findOne({ | ||
80 | include: [ | ||
81 | { | ||
82 | model: VideoModel.unscoped(), | ||
83 | required: true | ||
84 | }, | ||
85 | { | ||
86 | model: LocalVideoViewerWatchSectionModel.unscoped(), | ||
87 | required: true | ||
88 | } | ||
89 | ], | ||
90 | where: { | ||
91 | id | ||
92 | } | ||
93 | }) | ||
94 | } | ||
95 | |||
96 | static async getOverallStats (video: MVideo): Promise<VideoStatsOverall> { | ||
97 | const options = { | ||
98 | type: QueryTypes.SELECT as QueryTypes.SELECT, | ||
99 | replacements: { videoId: video.id } | ||
100 | } | ||
101 | |||
102 | const watchTimeQuery = `SELECT ` + | ||
103 | `SUM("localVideoViewer"."watchTime") AS "totalWatchTime", ` + | ||
104 | `AVG("localVideoViewer"."watchTime") AS "averageWatchTime" ` + | ||
105 | `FROM "localVideoViewer" ` + | ||
106 | `INNER JOIN "video" ON "video"."id" = "localVideoViewer"."videoId" ` + | ||
107 | `WHERE "videoId" = :videoId` | ||
108 | |||
109 | const watchTimePromise = LocalVideoViewerModel.sequelize.query<any>(watchTimeQuery, options) | ||
110 | |||
111 | const watchPeakQuery = `WITH "watchPeakValues" AS ( | ||
112 | SELECT "startDate" AS "dateBreakpoint", 1 AS "inc" | ||
113 | FROM "localVideoViewer" | ||
114 | WHERE "videoId" = :videoId | ||
115 | UNION ALL | ||
116 | SELECT "endDate" AS "dateBreakpoint", -1 AS "inc" | ||
117 | FROM "localVideoViewer" | ||
118 | WHERE "videoId" = :videoId | ||
119 | ) | ||
120 | SELECT "dateBreakpoint", "concurrent" | ||
121 | FROM ( | ||
122 | SELECT "dateBreakpoint", SUM(SUM("inc")) OVER (ORDER BY "dateBreakpoint") AS "concurrent" | ||
123 | FROM "watchPeakValues" | ||
124 | GROUP BY "dateBreakpoint" | ||
125 | ) tmp | ||
126 | ORDER BY "concurrent" DESC | ||
127 | FETCH FIRST 1 ROW ONLY` | ||
128 | const watchPeakPromise = LocalVideoViewerModel.sequelize.query<any>(watchPeakQuery, options) | ||
129 | |||
130 | const commentsQuery = `SELECT COUNT(*) AS comments FROM "videoComment" WHERE "videoId" = :videoId` | ||
131 | const commentsPromise = LocalVideoViewerModel.sequelize.query<any>(commentsQuery, options) | ||
132 | |||
133 | const countriesQuery = `SELECT country, COUNT(country) as viewers ` + | ||
134 | `FROM "localVideoViewer" ` + | ||
135 | `WHERE "videoId" = :videoId AND country IS NOT NULL ` + | ||
136 | `GROUP BY country ` + | ||
137 | `ORDER BY viewers DESC` | ||
138 | const countriesPromise = LocalVideoViewerModel.sequelize.query<any>(countriesQuery, options) | ||
139 | |||
140 | const [ rowsWatchTime, rowsWatchPeak, rowsComment, rowsCountries ] = await Promise.all([ | ||
141 | watchTimePromise, | ||
142 | watchPeakPromise, | ||
143 | commentsPromise, | ||
144 | countriesPromise | ||
145 | ]) | ||
146 | |||
147 | return { | ||
148 | totalWatchTime: rowsWatchTime.length !== 0 | ||
149 | ? Math.round(rowsWatchTime[0].totalWatchTime) || 0 | ||
150 | : 0, | ||
151 | averageWatchTime: rowsWatchTime.length !== 0 | ||
152 | ? Math.round(rowsWatchTime[0].averageWatchTime) || 0 | ||
153 | : 0, | ||
154 | |||
155 | viewersPeak: rowsWatchPeak.length !== 0 | ||
156 | ? parseInt(rowsWatchPeak[0].concurrent) || 0 | ||
157 | : 0, | ||
158 | viewersPeakDate: rowsWatchPeak.length !== 0 | ||
159 | ? rowsWatchPeak[0].dateBreakpoint || null | ||
160 | : null, | ||
161 | |||
162 | views: video.views, | ||
163 | likes: video.likes, | ||
164 | dislikes: video.dislikes, | ||
165 | |||
166 | comments: rowsComment.length !== 0 | ||
167 | ? parseInt(rowsComment[0].comments) || 0 | ||
168 | : 0, | ||
169 | |||
170 | countries: rowsCountries.map(r => ({ | ||
171 | isoCode: r.country, | ||
172 | viewers: r.viewers | ||
173 | })) | ||
174 | } | ||
175 | } | ||
176 | |||
177 | static async getRetentionStats (video: MVideo): Promise<VideoStatsRetention> { | ||
178 | const step = Math.max(Math.round(video.duration / 100), 1) | ||
179 | |||
180 | const query = `WITH "total" AS (SELECT COUNT(*) AS viewers FROM "localVideoViewer" WHERE "videoId" = :videoId) ` + | ||
181 | `SELECT serie AS "second", ` + | ||
182 | `(COUNT("localVideoViewer".id)::float / (SELECT GREATEST("total"."viewers", 1) FROM "total")) AS "retention" ` + | ||
183 | `FROM generate_series(0, ${video.duration}, ${step}) serie ` + | ||
184 | `LEFT JOIN "localVideoViewer" ON "localVideoViewer"."videoId" = :videoId ` + | ||
185 | `AND EXISTS (` + | ||
186 | `SELECT 1 FROM "localVideoViewerWatchSection" ` + | ||
187 | `WHERE "localVideoViewer"."id" = "localVideoViewerWatchSection"."localVideoViewerId" ` + | ||
188 | `AND serie >= "localVideoViewerWatchSection"."watchStart" ` + | ||
189 | `AND serie <= "localVideoViewerWatchSection"."watchEnd"` + | ||
190 | `)` + | ||
191 | `GROUP BY serie ` + | ||
192 | `ORDER BY serie ASC` | ||
193 | |||
194 | const queryOptions = { | ||
195 | type: QueryTypes.SELECT as QueryTypes.SELECT, | ||
196 | replacements: { videoId: video.id } | ||
197 | } | ||
198 | |||
199 | const rows = await LocalVideoViewerModel.sequelize.query<any>(query, queryOptions) | ||
200 | |||
201 | return { | ||
202 | data: rows.map(r => ({ | ||
203 | second: r.second, | ||
204 | retentionPercent: parseFloat(r.retention) * 100 | ||
205 | })) | ||
206 | } | ||
207 | } | ||
208 | |||
209 | static async getTimeserieStats (options: { | ||
210 | video: MVideo | ||
211 | metric: VideoStatsTimeserieMetric | ||
212 | }): Promise<VideoStatsTimeserie> { | ||
213 | const { video, metric } = options | ||
214 | |||
215 | const selectMetrics: { [ id in VideoStatsTimeserieMetric ]: string } = { | ||
216 | viewers: 'COUNT("localVideoViewer"."id")', | ||
217 | aggregateWatchTime: 'SUM("localVideoViewer"."watchTime")' | ||
218 | } | ||
219 | |||
220 | const query = `WITH days AS ( ` + | ||
221 | `SELECT (current_date::timestamp - (serie || ' days')::interval)::timestamptz AS day | ||
222 | FROM generate_series(0, ${STATS_TIMESERIE.MAX_DAYS - 1}) serie` + | ||
223 | `) ` + | ||
224 | `SELECT days.day AS date, COALESCE(${selectMetrics[metric]}, 0) AS value ` + | ||
225 | `FROM days ` + | ||
226 | `LEFT JOIN "localVideoViewer" ON "localVideoViewer"."videoId" = :videoId ` + | ||
227 | `AND date_trunc('day', "localVideoViewer"."startDate") = date_trunc('day', days.day) ` + | ||
228 | `GROUP BY day ` + | ||
229 | `ORDER BY day ` | ||
230 | |||
231 | const queryOptions = { | ||
232 | type: QueryTypes.SELECT as QueryTypes.SELECT, | ||
233 | replacements: { videoId: video.id } | ||
234 | } | ||
235 | |||
236 | const rows = await LocalVideoViewerModel.sequelize.query<any>(query, queryOptions) | ||
237 | |||
238 | return { | ||
239 | data: rows.map(r => ({ | ||
240 | date: r.date, | ||
241 | value: parseInt(r.value) | ||
242 | })) | ||
243 | } | ||
244 | } | ||
245 | |||
246 | toActivityPubObject (this: MLocalVideoViewerWithWatchSections): WatchActionObject { | ||
247 | const location = this.country | ||
248 | ? { | ||
249 | location: { | ||
250 | addressCountry: this.country | ||
251 | } | ||
252 | } | ||
253 | : {} | ||
254 | |||
255 | return { | ||
256 | id: this.url, | ||
257 | type: 'WatchAction', | ||
258 | duration: getActivityStreamDuration(this.watchTime), | ||
259 | startTime: this.startDate.toISOString(), | ||
260 | endTime: this.endDate.toISOString(), | ||
261 | |||
262 | object: this.Video.url, | ||
263 | uuid: this.uuid, | ||
264 | actionStatus: 'CompletedActionStatus', | ||
265 | |||
266 | watchSections: this.WatchSections.map(w => ({ | ||
267 | startTimestamp: w.watchStart, | ||
268 | endTimestamp: w.watchEnd | ||
269 | })), | ||
270 | |||
271 | ...location | ||
272 | } | ||
273 | } | ||
274 | } | ||
diff --git a/server/models/video/video-view.ts b/server/models/view/video-view.ts index d72df100f..df462e631 100644 --- a/server/models/video/video-view.ts +++ b/server/models/view/video-view.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { literal, Op } from 'sequelize' | 1 | import { literal, Op } from 'sequelize' |
2 | import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table } from 'sequelize-typescript' | 2 | import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table } from 'sequelize-typescript' |
3 | import { AttributesOnly } from '@shared/typescript-utils' | 3 | import { AttributesOnly } from '@shared/typescript-utils' |
4 | import { VideoModel } from './video' | 4 | import { VideoModel } from '../video/video' |
5 | 5 | ||
6 | @Table({ | 6 | @Table({ |
7 | tableName: 'videoView', | 7 | tableName: 'videoView', |
diff --git a/server/tests/api/activitypub/client.ts b/server/tests/api/activitypub/client.ts index e69ab3cb9..655fa30d0 100644 --- a/server/tests/api/activitypub/client.ts +++ b/server/tests/api/activitypub/client.ts | |||
@@ -2,6 +2,8 @@ | |||
2 | 2 | ||
3 | import 'mocha' | 3 | import 'mocha' |
4 | import * as chai from 'chai' | 4 | import * as chai from 'chai' |
5 | import { processViewersStats } from '@server/tests/shared' | ||
6 | import { HttpStatusCode, VideoPlaylistPrivacy, WatchActionObject } from '@shared/models' | ||
5 | import { | 7 | import { |
6 | cleanupTests, | 8 | cleanupTests, |
7 | createMultipleServers, | 9 | createMultipleServers, |
@@ -11,7 +13,6 @@ import { | |||
11 | setAccessTokensToServers, | 13 | setAccessTokensToServers, |
12 | setDefaultVideoChannel | 14 | setDefaultVideoChannel |
13 | } from '@shared/server-commands' | 15 | } from '@shared/server-commands' |
14 | import { HttpStatusCode, VideoPlaylistPrivacy } from '@shared/models' | ||
15 | 16 | ||
16 | const expect = chai.expect | 17 | const expect = chai.expect |
17 | 18 | ||
@@ -115,6 +116,23 @@ describe('Test activitypub', function () { | |||
115 | expect(res.header.location).to.equal('http://localhost:' + servers[0].port + '/videos/watch/' + video.uuid) | 116 | expect(res.header.location).to.equal('http://localhost:' + servers[0].port + '/videos/watch/' + video.uuid) |
116 | }) | 117 | }) |
117 | 118 | ||
119 | it('Should return the watch action', async function () { | ||
120 | this.timeout(50000) | ||
121 | |||
122 | await servers[0].views.simulateViewer({ id: video.uuid, currentTimes: [ 0, 2 ] }) | ||
123 | await processViewersStats(servers) | ||
124 | |||
125 | const res = await makeActivityPubGetRequest(servers[0].url, '/videos/local-viewer/1', HttpStatusCode.OK_200) | ||
126 | |||
127 | const object: WatchActionObject = res.body | ||
128 | expect(object.type).to.equal('WatchAction') | ||
129 | expect(object.duration).to.equal('PT2S') | ||
130 | expect(object.actionStatus).to.equal('CompletedActionStatus') | ||
131 | expect(object.watchSections).to.have.lengthOf(1) | ||
132 | expect(object.watchSections[0].startTimestamp).to.equal(0) | ||
133 | expect(object.watchSections[0].endTimestamp).to.equal(2) | ||
134 | }) | ||
135 | |||
118 | after(async function () { | 136 | after(async function () { |
119 | await cleanupTests(servers) | 137 | await cleanupTests(servers) |
120 | }) | 138 | }) |
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts index c9adeef4a..259d7e783 100644 --- a/server/tests/api/check-params/index.ts +++ b/server/tests/api/check-params/index.ts | |||
@@ -33,3 +33,4 @@ import './videos-common-filters' | |||
33 | import './video-files' | 33 | import './video-files' |
34 | import './videos-history' | 34 | import './videos-history' |
35 | import './videos-overviews' | 35 | import './videos-overviews' |
36 | import './views' | ||
diff --git a/server/tests/api/check-params/videos-history.ts b/server/tests/api/check-params/videos-history.ts index 82f38b7b4..c1b2d8bf3 100644 --- a/server/tests/api/check-params/videos-history.ts +++ b/server/tests/api/check-params/videos-history.ts | |||
@@ -17,7 +17,7 @@ import { | |||
17 | describe('Test videos history API validator', function () { | 17 | describe('Test videos history API validator', function () { |
18 | const myHistoryPath = '/api/v1/users/me/history/videos' | 18 | const myHistoryPath = '/api/v1/users/me/history/videos' |
19 | const myHistoryRemove = myHistoryPath + '/remove' | 19 | const myHistoryRemove = myHistoryPath + '/remove' |
20 | let watchingPath: string | 20 | let viewPath: string |
21 | let server: PeerTubeServer | 21 | let server: PeerTubeServer |
22 | let videoId: number | 22 | let videoId: number |
23 | 23 | ||
@@ -31,51 +31,15 @@ describe('Test videos history API validator', function () { | |||
31 | await setAccessTokensToServers([ server ]) | 31 | await setAccessTokensToServers([ server ]) |
32 | 32 | ||
33 | const { id, uuid } = await server.videos.upload() | 33 | const { id, uuid } = await server.videos.upload() |
34 | watchingPath = '/api/v1/videos/' + uuid + '/watching' | 34 | viewPath = '/api/v1/videos/' + uuid + '/views' |
35 | videoId = id | 35 | videoId = id |
36 | }) | 36 | }) |
37 | 37 | ||
38 | describe('When notifying a user is watching a video', function () { | 38 | describe('When notifying a user is watching a video', function () { |
39 | 39 | ||
40 | it('Should fail with an unauthenticated user', async function () { | 40 | it('Should fail with a bad token', async function () { |
41 | const fields = { currentTime: 5 } | ||
42 | await makePutBodyRequest({ url: server.url, path: watchingPath, fields, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) | ||
43 | }) | ||
44 | |||
45 | it('Should fail with an incorrect video id', async function () { | ||
46 | const fields = { currentTime: 5 } | 41 | const fields = { currentTime: 5 } |
47 | const path = '/api/v1/videos/blabla/watching' | 42 | await makePutBodyRequest({ url: server.url, path: viewPath, fields, token: 'bad', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) |
48 | await makePutBodyRequest({ | ||
49 | url: server.url, | ||
50 | path, | ||
51 | fields, | ||
52 | token: server.accessToken, | ||
53 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
54 | }) | ||
55 | }) | ||
56 | |||
57 | it('Should fail with an unknown video', async function () { | ||
58 | const fields = { currentTime: 5 } | ||
59 | const path = '/api/v1/videos/d91fff41-c24d-4508-8e13-3bd5902c3b02/watching' | ||
60 | |||
61 | await makePutBodyRequest({ | ||
62 | url: server.url, | ||
63 | path, | ||
64 | fields, | ||
65 | token: server.accessToken, | ||
66 | expectedStatus: HttpStatusCode.NOT_FOUND_404 | ||
67 | }) | ||
68 | }) | ||
69 | |||
70 | it('Should fail with a bad current time', async function () { | ||
71 | const fields = { currentTime: 'hello' } | ||
72 | await makePutBodyRequest({ | ||
73 | url: server.url, | ||
74 | path: watchingPath, | ||
75 | fields, | ||
76 | token: server.accessToken, | ||
77 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
78 | }) | ||
79 | }) | 43 | }) |
80 | 44 | ||
81 | it('Should succeed with the correct parameters', async function () { | 45 | it('Should succeed with the correct parameters', async function () { |
@@ -83,7 +47,7 @@ describe('Test videos history API validator', function () { | |||
83 | 47 | ||
84 | await makePutBodyRequest({ | 48 | await makePutBodyRequest({ |
85 | url: server.url, | 49 | url: server.url, |
86 | path: watchingPath, | 50 | path: viewPath, |
87 | fields, | 51 | fields, |
88 | token: server.accessToken, | 52 | token: server.accessToken, |
89 | expectedStatus: HttpStatusCode.NO_CONTENT_204 | 53 | expectedStatus: HttpStatusCode.NO_CONTENT_204 |
diff --git a/server/tests/api/check-params/views.ts b/server/tests/api/check-params/views.ts new file mode 100644 index 000000000..185b04af1 --- /dev/null +++ b/server/tests/api/check-params/views.ts | |||
@@ -0,0 +1,157 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import 'mocha' | ||
4 | import { HttpStatusCode, VideoPrivacy } from '@shared/models' | ||
5 | import { | ||
6 | cleanupTests, | ||
7 | createMultipleServers, | ||
8 | doubleFollow, | ||
9 | PeerTubeServer, | ||
10 | setAccessTokensToServers, | ||
11 | setDefaultVideoChannel | ||
12 | } from '@shared/server-commands' | ||
13 | |||
14 | describe('Test videos views', function () { | ||
15 | let servers: PeerTubeServer[] | ||
16 | let liveVideoId: string | ||
17 | let videoId: string | ||
18 | let remoteVideoId: string | ||
19 | let userAccessToken: string | ||
20 | |||
21 | before(async function () { | ||
22 | this.timeout(30000) | ||
23 | |||
24 | servers = await createMultipleServers(2) | ||
25 | await setAccessTokensToServers(servers) | ||
26 | await setDefaultVideoChannel(servers) | ||
27 | |||
28 | await servers[0].config.enableLive({ allowReplay: false, transcoding: false }); | ||
29 | |||
30 | ({ uuid: videoId } = await servers[0].videos.quickUpload({ name: 'video' })); | ||
31 | ({ uuid: remoteVideoId } = await servers[1].videos.quickUpload({ name: 'video' })); | ||
32 | ({ uuid: liveVideoId } = await servers[0].live.create({ | ||
33 | fields: { | ||
34 | name: 'live', | ||
35 | privacy: VideoPrivacy.PUBLIC, | ||
36 | channelId: servers[0].store.channel.id | ||
37 | } | ||
38 | })) | ||
39 | |||
40 | userAccessToken = await servers[0].users.generateUserAndToken('user') | ||
41 | |||
42 | await doubleFollow(servers[0], servers[1]) | ||
43 | }) | ||
44 | |||
45 | describe('When viewing a video', async function () { | ||
46 | |||
47 | // TODO: implement it when we'll remove backward compatibility in REST API | ||
48 | it('Should fail without current time') | ||
49 | |||
50 | it('Should fail with an invalid current time', async function () { | ||
51 | await servers[0].views.view({ id: videoId, currentTime: -1, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) | ||
52 | await servers[0].views.view({ id: videoId, currentTime: 10, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) | ||
53 | }) | ||
54 | |||
55 | it('Should succeed with correct parameters', async function () { | ||
56 | await servers[0].views.view({ id: videoId, currentTime: 1 }) | ||
57 | }) | ||
58 | }) | ||
59 | |||
60 | describe('When getting overall stats', function () { | ||
61 | |||
62 | it('Should fail with a remote video', async function () { | ||
63 | await servers[0].videoStats.getOverallStats({ videoId: remoteVideoId, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
64 | }) | ||
65 | |||
66 | it('Should fail without token', async function () { | ||
67 | await servers[0].videoStats.getOverallStats({ videoId: videoId, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) | ||
68 | }) | ||
69 | |||
70 | it('Should fail with another token', async function () { | ||
71 | await servers[0].videoStats.getOverallStats({ | ||
72 | videoId: videoId, | ||
73 | token: userAccessToken, | ||
74 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
75 | }) | ||
76 | }) | ||
77 | |||
78 | it('Should succeed with the correct parameters', async function () { | ||
79 | await servers[0].videoStats.getOverallStats({ videoId }) | ||
80 | }) | ||
81 | }) | ||
82 | |||
83 | describe('When getting timeserie stats', function () { | ||
84 | |||
85 | it('Should fail with a remote video', async function () { | ||
86 | await servers[0].videoStats.getTimeserieStats({ | ||
87 | videoId: remoteVideoId, | ||
88 | metric: 'viewers', | ||
89 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
90 | }) | ||
91 | }) | ||
92 | |||
93 | it('Should fail without token', async function () { | ||
94 | await servers[0].videoStats.getTimeserieStats({ | ||
95 | videoId: videoId, | ||
96 | token: null, | ||
97 | metric: 'viewers', | ||
98 | expectedStatus: HttpStatusCode.UNAUTHORIZED_401 | ||
99 | }) | ||
100 | }) | ||
101 | |||
102 | it('Should fail with another token', async function () { | ||
103 | await servers[0].videoStats.getTimeserieStats({ | ||
104 | videoId: videoId, | ||
105 | token: userAccessToken, | ||
106 | metric: 'viewers', | ||
107 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
108 | }) | ||
109 | }) | ||
110 | |||
111 | it('Should fail with an invalid metric', async function () { | ||
112 | await servers[0].videoStats.getTimeserieStats({ videoId, metric: 'hello' as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) | ||
113 | }) | ||
114 | |||
115 | it('Should succeed with the correct parameters', async function () { | ||
116 | await servers[0].videoStats.getTimeserieStats({ videoId, metric: 'viewers' }) | ||
117 | }) | ||
118 | }) | ||
119 | |||
120 | describe('When getting retention stats', function () { | ||
121 | |||
122 | it('Should fail with a remote video', async function () { | ||
123 | await servers[0].videoStats.getRetentionStats({ | ||
124 | videoId: remoteVideoId, | ||
125 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
126 | }) | ||
127 | }) | ||
128 | |||
129 | it('Should fail without token', async function () { | ||
130 | await servers[0].videoStats.getRetentionStats({ | ||
131 | videoId: videoId, | ||
132 | token: null, | ||
133 | expectedStatus: HttpStatusCode.UNAUTHORIZED_401 | ||
134 | }) | ||
135 | }) | ||
136 | |||
137 | it('Should fail with another token', async function () { | ||
138 | await servers[0].videoStats.getRetentionStats({ | ||
139 | videoId: videoId, | ||
140 | token: userAccessToken, | ||
141 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
142 | }) | ||
143 | }) | ||
144 | |||
145 | it('Should fail on live video', async function () { | ||
146 | await servers[0].videoStats.getRetentionStats({ videoId: liveVideoId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) | ||
147 | }) | ||
148 | |||
149 | it('Should succeed with the correct parameters', async function () { | ||
150 | await servers[0].videoStats.getRetentionStats({ videoId }) | ||
151 | }) | ||
152 | }) | ||
153 | |||
154 | after(async function () { | ||
155 | await cleanupTests(servers) | ||
156 | }) | ||
157 | }) | ||
diff --git a/server/tests/api/live/index.ts b/server/tests/api/live/index.ts index 105416b8d..71bc150d8 100644 --- a/server/tests/api/live/index.ts +++ b/server/tests/api/live/index.ts | |||
@@ -3,5 +3,4 @@ import './live-socket-messages' | |||
3 | import './live-permanent' | 3 | import './live-permanent' |
4 | import './live-rtmps' | 4 | import './live-rtmps' |
5 | import './live-save-replay' | 5 | import './live-save-replay' |
6 | import './live-views' | ||
7 | import './live' | 6 | import './live' |
diff --git a/server/tests/api/live/live-socket-messages.ts b/server/tests/api/live/live-socket-messages.ts index 50b16443e..7668ed5b9 100644 --- a/server/tests/api/live/live-socket-messages.ts +++ b/server/tests/api/live/live-socket-messages.ts | |||
@@ -140,8 +140,8 @@ describe('Test live', function () { | |||
140 | expect(localLastVideoViews).to.equal(0) | 140 | expect(localLastVideoViews).to.equal(0) |
141 | expect(remoteLastVideoViews).to.equal(0) | 141 | expect(remoteLastVideoViews).to.equal(0) |
142 | 142 | ||
143 | await servers[0].videos.view({ id: liveVideoUUID }) | 143 | await servers[0].views.simulateView({ id: liveVideoUUID }) |
144 | await servers[1].videos.view({ id: liveVideoUUID }) | 144 | await servers[1].views.simulateView({ id: liveVideoUUID }) |
145 | 145 | ||
146 | await waitJobs(servers) | 146 | await waitJobs(servers) |
147 | 147 | ||
diff --git a/server/tests/api/live/live-views.ts b/server/tests/api/live/live-views.ts deleted file mode 100644 index 446d0913c..000000000 --- a/server/tests/api/live/live-views.ts +++ /dev/null | |||
@@ -1,132 +0,0 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import 'mocha' | ||
4 | import * as chai from 'chai' | ||
5 | import { FfmpegCommand } from 'fluent-ffmpeg' | ||
6 | import { wait } from '@shared/core-utils' | ||
7 | import { VideoPrivacy } from '@shared/models' | ||
8 | import { | ||
9 | cleanupTests, | ||
10 | createMultipleServers, | ||
11 | doubleFollow, | ||
12 | PeerTubeServer, | ||
13 | setAccessTokensToServers, | ||
14 | setDefaultVideoChannel, | ||
15 | stopFfmpeg, | ||
16 | waitJobs, | ||
17 | waitUntilLivePublishedOnAllServers | ||
18 | } from '@shared/server-commands' | ||
19 | |||
20 | const expect = chai.expect | ||
21 | |||
22 | describe('Live views', function () { | ||
23 | let servers: PeerTubeServer[] = [] | ||
24 | |||
25 | before(async function () { | ||
26 | this.timeout(120000) | ||
27 | |||
28 | servers = await createMultipleServers(2) | ||
29 | |||
30 | // Get the access tokens | ||
31 | await setAccessTokensToServers(servers) | ||
32 | await setDefaultVideoChannel(servers) | ||
33 | |||
34 | await servers[0].config.updateCustomSubConfig({ | ||
35 | newConfig: { | ||
36 | live: { | ||
37 | enabled: true, | ||
38 | allowReplay: true, | ||
39 | transcoding: { | ||
40 | enabled: false | ||
41 | } | ||
42 | } | ||
43 | } | ||
44 | }) | ||
45 | |||
46 | // Server 1 and server 2 follow each other | ||
47 | await doubleFollow(servers[0], servers[1]) | ||
48 | }) | ||
49 | |||
50 | let liveVideoId: string | ||
51 | let command: FfmpegCommand | ||
52 | |||
53 | async function countViewers (expectedViewers: number) { | ||
54 | for (const server of servers) { | ||
55 | const video = await server.videos.get({ id: liveVideoId }) | ||
56 | expect(video.viewers).to.equal(expectedViewers) | ||
57 | } | ||
58 | } | ||
59 | |||
60 | async function countViews (expectedViews: number) { | ||
61 | for (const server of servers) { | ||
62 | const video = await server.videos.get({ id: liveVideoId }) | ||
63 | expect(video.views).to.equal(expectedViews) | ||
64 | } | ||
65 | } | ||
66 | |||
67 | before(async function () { | ||
68 | this.timeout(30000) | ||
69 | |||
70 | const liveAttributes = { | ||
71 | name: 'live video', | ||
72 | channelId: servers[0].store.channel.id, | ||
73 | privacy: VideoPrivacy.PUBLIC | ||
74 | } | ||
75 | |||
76 | const live = await servers[0].live.create({ fields: liveAttributes }) | ||
77 | liveVideoId = live.uuid | ||
78 | |||
79 | command = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoId }) | ||
80 | await waitUntilLivePublishedOnAllServers(servers, liveVideoId) | ||
81 | await waitJobs(servers) | ||
82 | }) | ||
83 | |||
84 | it('Should display no views and viewers for a live', async function () { | ||
85 | await countViews(0) | ||
86 | await countViewers(0) | ||
87 | }) | ||
88 | |||
89 | it('Should view a live twice and display 1 view/viewer', async function () { | ||
90 | this.timeout(30000) | ||
91 | |||
92 | await servers[0].videos.view({ id: liveVideoId }) | ||
93 | await servers[0].videos.view({ id: liveVideoId }) | ||
94 | |||
95 | await waitJobs(servers) | ||
96 | await countViewers(1) | ||
97 | |||
98 | await wait(7000) | ||
99 | await countViews(1) | ||
100 | }) | ||
101 | |||
102 | it('Should wait and display 0 viewers while still have 1 view', async function () { | ||
103 | this.timeout(30000) | ||
104 | |||
105 | await wait(12000) | ||
106 | await waitJobs(servers) | ||
107 | |||
108 | await countViews(1) | ||
109 | await countViewers(0) | ||
110 | }) | ||
111 | |||
112 | it('Should view a live on a remote and on local and display 2 viewers and 3 views', async function () { | ||
113 | this.timeout(30000) | ||
114 | |||
115 | await servers[0].videos.view({ id: liveVideoId }) | ||
116 | await servers[1].videos.view({ id: liveVideoId }) | ||
117 | await servers[1].videos.view({ id: liveVideoId }) | ||
118 | await waitJobs(servers) | ||
119 | |||
120 | await countViewers(2) | ||
121 | |||
122 | await wait(7000) | ||
123 | await waitJobs(servers) | ||
124 | |||
125 | await countViews(3) | ||
126 | }) | ||
127 | |||
128 | after(async function () { | ||
129 | await stopFfmpeg(command) | ||
130 | await cleanupTests(servers) | ||
131 | }) | ||
132 | }) | ||
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index 3f2286278..0f7ffcb4c 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts | |||
@@ -87,7 +87,7 @@ async function createServers (strategy: VideoRedundancyStrategy | null, addition | |||
87 | const { id } = await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } }) | 87 | const { id } = await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } }) |
88 | video1Server2 = await servers[1].videos.get({ id }) | 88 | video1Server2 = await servers[1].videos.get({ id }) |
89 | 89 | ||
90 | await servers[1].videos.view({ id }) | 90 | await servers[1].views.simulateView({ id }) |
91 | } | 91 | } |
92 | 92 | ||
93 | await waitJobs(servers) | 93 | await waitJobs(servers) |
@@ -447,8 +447,8 @@ describe('Test videos redundancy', function () { | |||
447 | it('Should view 2 times the first video to have > min_views config', async function () { | 447 | it('Should view 2 times the first video to have > min_views config', async function () { |
448 | this.timeout(80000) | 448 | this.timeout(80000) |
449 | 449 | ||
450 | await servers[0].videos.view({ id: video1Server2.uuid }) | 450 | await servers[0].views.simulateView({ id: video1Server2.uuid }) |
451 | await servers[2].videos.view({ id: video1Server2.uuid }) | 451 | await servers[2].views.simulateView({ id: video1Server2.uuid }) |
452 | 452 | ||
453 | await wait(10000) | 453 | await wait(10000) |
454 | await waitJobs(servers) | 454 | await waitJobs(servers) |
@@ -516,8 +516,8 @@ describe('Test videos redundancy', function () { | |||
516 | it('Should have 1 redundancy on the first video', async function () { | 516 | it('Should have 1 redundancy on the first video', async function () { |
517 | this.timeout(160000) | 517 | this.timeout(160000) |
518 | 518 | ||
519 | await servers[0].videos.view({ id: video1Server2.uuid }) | 519 | await servers[0].views.simulateView({ id: video1Server2.uuid }) |
520 | await servers[2].videos.view({ id: video1Server2.uuid }) | 520 | await servers[2].views.simulateView({ id: video1Server2.uuid }) |
521 | 521 | ||
522 | await wait(10000) | 522 | await wait(10000) |
523 | await waitJobs(servers) | 523 | await waitJobs(servers) |
diff --git a/server/tests/api/server/reverse-proxy.ts b/server/tests/api/server/reverse-proxy.ts index 968d98e96..fa2063536 100644 --- a/server/tests/api/server/reverse-proxy.ts +++ b/server/tests/api/server/reverse-proxy.ts | |||
@@ -41,8 +41,8 @@ describe('Test application behind a reverse proxy', function () { | |||
41 | it('Should view a video only once with the same IP by default', async function () { | 41 | it('Should view a video only once with the same IP by default', async function () { |
42 | this.timeout(20000) | 42 | this.timeout(20000) |
43 | 43 | ||
44 | await server.videos.view({ id: videoId }) | 44 | await server.views.simulateView({ id: videoId }) |
45 | await server.videos.view({ id: videoId }) | 45 | await server.views.simulateView({ id: videoId }) |
46 | 46 | ||
47 | // Wait the repeatable job | 47 | // Wait the repeatable job |
48 | await wait(8000) | 48 | await wait(8000) |
@@ -54,8 +54,8 @@ describe('Test application behind a reverse proxy', function () { | |||
54 | it('Should view a video 2 times with the X-Forwarded-For header set', async function () { | 54 | it('Should view a video 2 times with the X-Forwarded-For header set', async function () { |
55 | this.timeout(20000) | 55 | this.timeout(20000) |
56 | 56 | ||
57 | await server.videos.view({ id: videoId, xForwardedFor: '0.0.0.1,127.0.0.1' }) | 57 | await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.1,127.0.0.1' }) |
58 | await server.videos.view({ id: videoId, xForwardedFor: '0.0.0.2,127.0.0.1' }) | 58 | await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.2,127.0.0.1' }) |
59 | 59 | ||
60 | // Wait the repeatable job | 60 | // Wait the repeatable job |
61 | await wait(8000) | 61 | await wait(8000) |
@@ -67,8 +67,8 @@ describe('Test application behind a reverse proxy', function () { | |||
67 | it('Should view a video only once with the same client IP in the X-Forwarded-For header', async function () { | 67 | it('Should view a video only once with the same client IP in the X-Forwarded-For header', async function () { |
68 | this.timeout(20000) | 68 | this.timeout(20000) |
69 | 69 | ||
70 | await server.videos.view({ id: videoId, xForwardedFor: '0.0.0.4,0.0.0.3,::ffff:127.0.0.1' }) | 70 | await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.4,0.0.0.3,::ffff:127.0.0.1' }) |
71 | await server.videos.view({ id: videoId, xForwardedFor: '0.0.0.5,0.0.0.3,127.0.0.1' }) | 71 | await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.5,0.0.0.3,127.0.0.1' }) |
72 | 72 | ||
73 | // Wait the repeatable job | 73 | // Wait the repeatable job |
74 | await wait(8000) | 74 | await wait(8000) |
@@ -80,8 +80,8 @@ describe('Test application behind a reverse proxy', function () { | |||
80 | it('Should view a video two times with a different client IP in the X-Forwarded-For header', async function () { | 80 | it('Should view a video two times with a different client IP in the X-Forwarded-For header', async function () { |
81 | this.timeout(20000) | 81 | this.timeout(20000) |
82 | 82 | ||
83 | await server.videos.view({ id: videoId, xForwardedFor: '0.0.0.8,0.0.0.6,127.0.0.1' }) | 83 | await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.8,0.0.0.6,127.0.0.1' }) |
84 | await server.videos.view({ id: videoId, xForwardedFor: '0.0.0.8,0.0.0.7,127.0.0.1' }) | 84 | await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.8,0.0.0.7,127.0.0.1' }) |
85 | 85 | ||
86 | // Wait the repeatable job | 86 | // Wait the repeatable job |
87 | await wait(8000) | 87 | await wait(8000) |
diff --git a/server/tests/api/server/stats.ts b/server/tests/api/server/stats.ts index 2296c0cb9..a9ae236fb 100644 --- a/server/tests/api/server/stats.ts +++ b/server/tests/api/server/stats.ts | |||
@@ -38,7 +38,7 @@ describe('Test stats (excluding redundancy)', function () { | |||
38 | 38 | ||
39 | await servers[0].comments.createThread({ videoId: uuid, text: 'comment' }) | 39 | await servers[0].comments.createThread({ videoId: uuid, text: 'comment' }) |
40 | 40 | ||
41 | await servers[0].videos.view({ id: uuid }) | 41 | await servers[0].views.simulateView({ id: uuid }) |
42 | 42 | ||
43 | // Wait the video views repeatable job | 43 | // Wait the video views repeatable job |
44 | await wait(8000) | 44 | await wait(8000) |
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts index 7dc826353..27b119f30 100644 --- a/server/tests/api/videos/index.ts +++ b/server/tests/api/videos/index.ts | |||
@@ -16,4 +16,3 @@ import './video-schedule-update' | |||
16 | import './videos-common-filters' | 16 | import './videos-common-filters' |
17 | import './videos-history' | 17 | import './videos-history' |
18 | import './videos-overview' | 18 | import './videos-overview' |
19 | import './videos-views-cleaner' | ||
diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts index a9df262dc..84c1515a3 100644 --- a/server/tests/api/videos/multiple-servers.ts +++ b/server/tests/api/videos/multiple-servers.ts | |||
@@ -504,21 +504,22 @@ describe('Test multiple servers', function () { | |||
504 | it('Should view multiple videos on owned servers', async function () { | 504 | it('Should view multiple videos on owned servers', async function () { |
505 | this.timeout(30000) | 505 | this.timeout(30000) |
506 | 506 | ||
507 | await servers[2].videos.view({ id: localVideosServer3[0] }) | 507 | await servers[2].views.simulateView({ id: localVideosServer3[0] }) |
508 | await wait(1000) | 508 | await wait(1000) |
509 | 509 | ||
510 | await servers[2].videos.view({ id: localVideosServer3[0] }) | 510 | await servers[2].views.simulateView({ id: localVideosServer3[0] }) |
511 | await servers[2].videos.view({ id: localVideosServer3[1] }) | 511 | await servers[2].views.simulateView({ id: localVideosServer3[1] }) |
512 | 512 | ||
513 | await wait(1000) | 513 | await wait(1000) |
514 | 514 | ||
515 | await servers[2].videos.view({ id: localVideosServer3[0] }) | 515 | await servers[2].views.simulateView({ id: localVideosServer3[0] }) |
516 | await servers[2].videos.view({ id: localVideosServer3[0] }) | 516 | await servers[2].views.simulateView({ id: localVideosServer3[0] }) |
517 | 517 | ||
518 | await waitJobs(servers) | 518 | await waitJobs(servers) |
519 | 519 | ||
520 | // Wait the repeatable job | 520 | for (const server of servers) { |
521 | await wait(6000) | 521 | await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } }) |
522 | } | ||
522 | 523 | ||
523 | await waitJobs(servers) | 524 | await waitJobs(servers) |
524 | 525 | ||
@@ -537,23 +538,24 @@ describe('Test multiple servers', function () { | |||
537 | this.timeout(45000) | 538 | this.timeout(45000) |
538 | 539 | ||
539 | const tasks: Promise<any>[] = [] | 540 | const tasks: Promise<any>[] = [] |
540 | tasks.push(servers[0].videos.view({ id: remoteVideosServer1[0] })) | 541 | tasks.push(servers[0].views.simulateView({ id: remoteVideosServer1[0] })) |
541 | tasks.push(servers[1].videos.view({ id: remoteVideosServer2[0] })) | 542 | tasks.push(servers[1].views.simulateView({ id: remoteVideosServer2[0] })) |
542 | tasks.push(servers[1].videos.view({ id: remoteVideosServer2[0] })) | 543 | tasks.push(servers[1].views.simulateView({ id: remoteVideosServer2[0] })) |
543 | tasks.push(servers[2].videos.view({ id: remoteVideosServer3[0] })) | 544 | tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[0] })) |
544 | tasks.push(servers[2].videos.view({ id: remoteVideosServer3[1] })) | 545 | tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[1] })) |
545 | tasks.push(servers[2].videos.view({ id: remoteVideosServer3[1] })) | 546 | tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[1] })) |
546 | tasks.push(servers[2].videos.view({ id: remoteVideosServer3[1] })) | 547 | tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[1] })) |
547 | tasks.push(servers[2].videos.view({ id: localVideosServer3[1] })) | 548 | tasks.push(servers[2].views.simulateView({ id: localVideosServer3[1] })) |
548 | tasks.push(servers[2].videos.view({ id: localVideosServer3[1] })) | 549 | tasks.push(servers[2].views.simulateView({ id: localVideosServer3[1] })) |
549 | tasks.push(servers[2].videos.view({ id: localVideosServer3[1] })) | 550 | tasks.push(servers[2].views.simulateView({ id: localVideosServer3[1] })) |
550 | 551 | ||
551 | await Promise.all(tasks) | 552 | await Promise.all(tasks) |
552 | 553 | ||
553 | await waitJobs(servers) | 554 | await waitJobs(servers) |
554 | 555 | ||
555 | // Wait the repeatable job | 556 | for (const server of servers) { |
556 | await wait(16000) | 557 | await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } }) |
558 | } | ||
557 | 559 | ||
558 | await waitJobs(servers) | 560 | await waitJobs(servers) |
559 | 561 | ||
diff --git a/server/tests/api/videos/single-server.ts b/server/tests/api/videos/single-server.ts index d37043aef..0e429fef7 100644 --- a/server/tests/api/videos/single-server.ts +++ b/server/tests/api/videos/single-server.ts | |||
@@ -179,22 +179,21 @@ describe('Test a single server', function () { | |||
179 | it('Should have the views updated', async function () { | 179 | it('Should have the views updated', async function () { |
180 | this.timeout(20000) | 180 | this.timeout(20000) |
181 | 181 | ||
182 | await server.videos.view({ id: videoId }) | 182 | await server.views.simulateView({ id: videoId }) |
183 | await server.videos.view({ id: videoId }) | 183 | await server.views.simulateView({ id: videoId }) |
184 | await server.videos.view({ id: videoId }) | 184 | await server.views.simulateView({ id: videoId }) |
185 | 185 | ||
186 | await wait(1500) | 186 | await wait(1500) |
187 | 187 | ||
188 | await server.videos.view({ id: videoId }) | 188 | await server.views.simulateView({ id: videoId }) |
189 | await server.videos.view({ id: videoId }) | 189 | await server.views.simulateView({ id: videoId }) |
190 | 190 | ||
191 | await wait(1500) | 191 | await wait(1500) |
192 | 192 | ||
193 | await server.videos.view({ id: videoId }) | 193 | await server.views.simulateView({ id: videoId }) |
194 | await server.videos.view({ id: videoId }) | 194 | await server.views.simulateView({ id: videoId }) |
195 | 195 | ||
196 | // Wait the repeatable job | 196 | await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } }) |
197 | await wait(8000) | ||
198 | 197 | ||
199 | const video = await server.videos.get({ id: videoId }) | 198 | const video = await server.videos.get({ id: videoId }) |
200 | expect(video.views).to.equal(3) | 199 | expect(video.views).to.equal(3) |
diff --git a/server/tests/api/videos/video-channels.ts b/server/tests/api/videos/video-channels.ts index 09a4bfa70..6f495c42d 100644 --- a/server/tests/api/videos/video-channels.ts +++ b/server/tests/api/videos/video-channels.ts | |||
@@ -466,8 +466,8 @@ describe('Test video channels', function () { | |||
466 | 466 | ||
467 | { | 467 | { |
468 | // video has been posted on channel servers[0].store.videoChannel.id since last update | 468 | // video has been posted on channel servers[0].store.videoChannel.id since last update |
469 | await servers[0].videos.view({ id: videoUUID, xForwardedFor: '0.0.0.1,127.0.0.1' }) | 469 | await servers[0].views.simulateView({ id: videoUUID, xForwardedFor: '0.0.0.1,127.0.0.1' }) |
470 | await servers[0].videos.view({ id: videoUUID, xForwardedFor: '0.0.0.2,127.0.0.1' }) | 470 | await servers[0].views.simulateView({ id: videoUUID, xForwardedFor: '0.0.0.2,127.0.0.1' }) |
471 | 471 | ||
472 | // Wait the repeatable job | 472 | // Wait the repeatable job |
473 | await wait(8000) | 473 | await wait(8000) |
diff --git a/server/tests/api/videos/videos-history.ts b/server/tests/api/videos/videos-history.ts index 8648c97f0..b1b3ff10a 100644 --- a/server/tests/api/videos/videos-history.ts +++ b/server/tests/api/videos/videos-history.ts | |||
@@ -3,15 +3,8 @@ | |||
3 | import 'mocha' | 3 | import 'mocha' |
4 | import * as chai from 'chai' | 4 | import * as chai from 'chai' |
5 | import { wait } from '@shared/core-utils' | 5 | import { wait } from '@shared/core-utils' |
6 | import { HttpStatusCode, Video } from '@shared/models' | 6 | import { Video } from '@shared/models' |
7 | import { | 7 | import { cleanupTests, createSingleServer, killallServers, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' |
8 | cleanupTests, | ||
9 | createSingleServer, | ||
10 | HistoryCommand, | ||
11 | killallServers, | ||
12 | PeerTubeServer, | ||
13 | setAccessTokensToServers | ||
14 | } from '@shared/server-commands' | ||
15 | 8 | ||
16 | const expect = chai.expect | 9 | const expect = chai.expect |
17 | 10 | ||
@@ -23,7 +16,6 @@ describe('Test videos history', function () { | |||
23 | let video3UUID: string | 16 | let video3UUID: string |
24 | let video3WatchedDate: Date | 17 | let video3WatchedDate: Date |
25 | let userAccessToken: string | 18 | let userAccessToken: string |
26 | let command: HistoryCommand | ||
27 | 19 | ||
28 | before(async function () { | 20 | before(async function () { |
29 | this.timeout(30000) | 21 | this.timeout(30000) |
@@ -32,30 +24,26 @@ describe('Test videos history', function () { | |||
32 | 24 | ||
33 | await setAccessTokensToServers([ server ]) | 25 | await setAccessTokensToServers([ server ]) |
34 | 26 | ||
35 | command = server.history | 27 | // 10 seconds long |
28 | const fixture = 'video_59fps.mp4' | ||
36 | 29 | ||
37 | { | 30 | { |
38 | const { id, uuid } = await server.videos.upload({ attributes: { name: 'video 1' } }) | 31 | const { id, uuid } = await server.videos.upload({ attributes: { name: 'video 1', fixture } }) |
39 | video1UUID = uuid | 32 | video1UUID = uuid |
40 | video1Id = id | 33 | video1Id = id |
41 | } | 34 | } |
42 | 35 | ||
43 | { | 36 | { |
44 | const { uuid } = await server.videos.upload({ attributes: { name: 'video 2' } }) | 37 | const { uuid } = await server.videos.upload({ attributes: { name: 'video 2', fixture } }) |
45 | video2UUID = uuid | 38 | video2UUID = uuid |
46 | } | 39 | } |
47 | 40 | ||
48 | { | 41 | { |
49 | const { uuid } = await server.videos.upload({ attributes: { name: 'video 3' } }) | 42 | const { uuid } = await server.videos.upload({ attributes: { name: 'video 3', fixture } }) |
50 | video3UUID = uuid | 43 | video3UUID = uuid |
51 | } | 44 | } |
52 | 45 | ||
53 | const user = { | 46 | userAccessToken = await server.users.generateUserAndToken('user_1') |
54 | username: 'user_1', | ||
55 | password: 'super password' | ||
56 | } | ||
57 | await server.users.create({ username: user.username, password: user.password }) | ||
58 | userAccessToken = await server.login.getAccessToken(user) | ||
59 | }) | 47 | }) |
60 | 48 | ||
61 | it('Should get videos, without watching history', async function () { | 49 | it('Should get videos, without watching history', async function () { |
@@ -70,8 +58,8 @@ describe('Test videos history', function () { | |||
70 | }) | 58 | }) |
71 | 59 | ||
72 | it('Should watch the first and second video', async function () { | 60 | it('Should watch the first and second video', async function () { |
73 | await command.watchVideo({ videoId: video2UUID, currentTime: 8 }) | 61 | await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 }) |
74 | await command.watchVideo({ videoId: video1UUID, currentTime: 3 }) | 62 | await server.views.view({ id: video1UUID, token: server.accessToken, currentTime: 3 }) |
75 | }) | 63 | }) |
76 | 64 | ||
77 | it('Should return the correct history when listing, searching and getting videos', async function () { | 65 | it('Should return the correct history when listing, searching and getting videos', async function () { |
@@ -124,9 +112,9 @@ describe('Test videos history', function () { | |||
124 | 112 | ||
125 | it('Should have these videos when listing my history', async function () { | 113 | it('Should have these videos when listing my history', async function () { |
126 | video3WatchedDate = new Date() | 114 | video3WatchedDate = new Date() |
127 | await command.watchVideo({ videoId: video3UUID, currentTime: 2 }) | 115 | await server.views.view({ id: video3UUID, token: server.accessToken, currentTime: 2 }) |
128 | 116 | ||
129 | const body = await command.list() | 117 | const body = await server.history.list() |
130 | 118 | ||
131 | expect(body.total).to.equal(3) | 119 | expect(body.total).to.equal(3) |
132 | 120 | ||
@@ -137,14 +125,14 @@ describe('Test videos history', function () { | |||
137 | }) | 125 | }) |
138 | 126 | ||
139 | it('Should not have videos history on another user', async function () { | 127 | it('Should not have videos history on another user', async function () { |
140 | const body = await command.list({ token: userAccessToken }) | 128 | const body = await server.history.list({ token: userAccessToken }) |
141 | 129 | ||
142 | expect(body.total).to.equal(0) | 130 | expect(body.total).to.equal(0) |
143 | expect(body.data).to.have.lengthOf(0) | 131 | expect(body.data).to.have.lengthOf(0) |
144 | }) | 132 | }) |
145 | 133 | ||
146 | it('Should be able to search through videos in my history', async function () { | 134 | it('Should be able to search through videos in my history', async function () { |
147 | const body = await command.list({ search: '2' }) | 135 | const body = await server.history.list({ search: '2' }) |
148 | expect(body.total).to.equal(1) | 136 | expect(body.total).to.equal(1) |
149 | 137 | ||
150 | const videos = body.data | 138 | const videos = body.data |
@@ -152,11 +140,11 @@ describe('Test videos history', function () { | |||
152 | }) | 140 | }) |
153 | 141 | ||
154 | it('Should clear my history', async function () { | 142 | it('Should clear my history', async function () { |
155 | await command.removeAll({ beforeDate: video3WatchedDate.toISOString() }) | 143 | await server.history.removeAll({ beforeDate: video3WatchedDate.toISOString() }) |
156 | }) | 144 | }) |
157 | 145 | ||
158 | it('Should have my history cleared', async function () { | 146 | it('Should have my history cleared', async function () { |
159 | const body = await command.list() | 147 | const body = await server.history.list() |
160 | expect(body.total).to.equal(1) | 148 | expect(body.total).to.equal(1) |
161 | 149 | ||
162 | const videos = body.data | 150 | const videos = body.data |
@@ -168,7 +156,10 @@ describe('Test videos history', function () { | |||
168 | videosHistoryEnabled: false | 156 | videosHistoryEnabled: false |
169 | }) | 157 | }) |
170 | 158 | ||
171 | await command.watchVideo({ videoId: video2UUID, currentTime: 8, expectedStatus: HttpStatusCode.CONFLICT_409 }) | 159 | await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 }) |
160 | |||
161 | const { data } = await server.history.list() | ||
162 | expect(data[0].name).to.not.equal('video 2') | ||
172 | }) | 163 | }) |
173 | 164 | ||
174 | it('Should re-enable videos history', async function () { | 165 | it('Should re-enable videos history', async function () { |
@@ -176,14 +167,10 @@ describe('Test videos history', function () { | |||
176 | videosHistoryEnabled: true | 167 | videosHistoryEnabled: true |
177 | }) | 168 | }) |
178 | 169 | ||
179 | await command.watchVideo({ videoId: video1UUID, currentTime: 8 }) | 170 | await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 }) |
180 | 171 | ||
181 | const body = await command.list() | 172 | const { data } = await server.history.list() |
182 | expect(body.total).to.equal(2) | 173 | expect(data[0].name).to.equal('video 2') |
183 | |||
184 | const videos = body.data | ||
185 | expect(videos[0].name).to.equal('video 1') | ||
186 | expect(videos[1].name).to.equal('video 3') | ||
187 | }) | 174 | }) |
188 | 175 | ||
189 | it('Should not clean old history', async function () { | 176 | it('Should not clean old history', async function () { |
@@ -197,7 +184,7 @@ describe('Test videos history', function () { | |||
197 | 184 | ||
198 | // Should still have history | 185 | // Should still have history |
199 | 186 | ||
200 | const body = await command.list() | 187 | const body = await server.history.list() |
201 | expect(body.total).to.equal(2) | 188 | expect(body.total).to.equal(2) |
202 | }) | 189 | }) |
203 | 190 | ||
@@ -210,25 +197,25 @@ describe('Test videos history', function () { | |||
210 | 197 | ||
211 | await wait(6000) | 198 | await wait(6000) |
212 | 199 | ||
213 | const body = await command.list() | 200 | const body = await server.history.list() |
214 | expect(body.total).to.equal(0) | 201 | expect(body.total).to.equal(0) |
215 | }) | 202 | }) |
216 | 203 | ||
217 | it('Should delete a specific history element', async function () { | 204 | it('Should delete a specific history element', async function () { |
218 | { | 205 | { |
219 | await command.watchVideo({ videoId: video1UUID, currentTime: 4 }) | 206 | await server.views.view({ id: video1UUID, token: server.accessToken, currentTime: 4 }) |
220 | await command.watchVideo({ videoId: video2UUID, currentTime: 8 }) | 207 | await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 }) |
221 | } | 208 | } |
222 | 209 | ||
223 | { | 210 | { |
224 | const body = await command.list() | 211 | const body = await server.history.list() |
225 | expect(body.total).to.equal(2) | 212 | expect(body.total).to.equal(2) |
226 | } | 213 | } |
227 | 214 | ||
228 | { | 215 | { |
229 | await command.removeElement({ videoId: video1Id }) | 216 | await server.history.removeElement({ videoId: video1Id }) |
230 | 217 | ||
231 | const body = await command.list() | 218 | const body = await server.history.list() |
232 | expect(body.total).to.equal(1) | 219 | expect(body.total).to.equal(1) |
233 | expect(body.data[0].uuid).to.equal(video2UUID) | 220 | expect(body.data[0].uuid).to.equal(video2UUID) |
234 | } | 221 | } |
diff --git a/server/tests/api/views/index.ts b/server/tests/api/views/index.ts new file mode 100644 index 000000000..5e06b31fb --- /dev/null +++ b/server/tests/api/views/index.ts | |||
@@ -0,0 +1,5 @@ | |||
1 | export * from './video-views-counter' | ||
2 | export * from './video-views-overall-stats' | ||
3 | export * from './video-views-retention-stats' | ||
4 | export * from './video-views-timeserie-stats' | ||
5 | export * from './videos-views-cleaner' | ||
diff --git a/server/tests/api/views/video-views-counter.ts b/server/tests/api/views/video-views-counter.ts new file mode 100644 index 000000000..b68aaa350 --- /dev/null +++ b/server/tests/api/views/video-views-counter.ts | |||
@@ -0,0 +1,155 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import 'mocha' | ||
4 | import * as chai from 'chai' | ||
5 | import { FfmpegCommand } from 'fluent-ffmpeg' | ||
6 | import { prepareViewsServers, prepareViewsVideos, processViewsBuffer } from '@server/tests/shared' | ||
7 | import { wait } from '@shared/core-utils' | ||
8 | import { cleanupTests, PeerTubeServer, stopFfmpeg, waitJobs } from '@shared/server-commands' | ||
9 | |||
10 | const expect = chai.expect | ||
11 | |||
12 | describe('Test video views/viewers counters', function () { | ||
13 | let servers: PeerTubeServer[] | ||
14 | |||
15 | async function checkCounter (field: 'views' | 'viewers', id: string, expected: number) { | ||
16 | for (const server of servers) { | ||
17 | const video = await server.videos.get({ id }) | ||
18 | |||
19 | const messageSuffix = video.isLive | ||
20 | ? 'live video' | ||
21 | : 'vod video' | ||
22 | |||
23 | expect(video[field]).to.equal(expected, `${field} not valid on server ${server.serverNumber} for ${messageSuffix} ${video.uuid}`) | ||
24 | } | ||
25 | } | ||
26 | |||
27 | before(async function () { | ||
28 | this.timeout(120000) | ||
29 | |||
30 | servers = await prepareViewsServers() | ||
31 | }) | ||
32 | |||
33 | describe('Test views counter on VOD', function () { | ||
34 | let videoUUID: string | ||
35 | |||
36 | before(async function () { | ||
37 | this.timeout(30000) | ||
38 | |||
39 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) | ||
40 | videoUUID = uuid | ||
41 | |||
42 | await waitJobs(servers) | ||
43 | }) | ||
44 | |||
45 | it('Should not view a video if watch time is below the threshold', async function () { | ||
46 | await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 2 ] }) | ||
47 | await processViewsBuffer(servers) | ||
48 | |||
49 | await checkCounter('views', videoUUID, 0) | ||
50 | }) | ||
51 | |||
52 | it('Should view a video if watch time is above the threshold', async function () { | ||
53 | await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 4 ] }) | ||
54 | await processViewsBuffer(servers) | ||
55 | |||
56 | await checkCounter('views', videoUUID, 1) | ||
57 | }) | ||
58 | |||
59 | it('Should not view again this video with the same IP', async function () { | ||
60 | await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 4 ] }) | ||
61 | await processViewsBuffer(servers) | ||
62 | |||
63 | await checkCounter('views', videoUUID, 1) | ||
64 | }) | ||
65 | |||
66 | it('Should view the video from server 2 and send the event', async function () { | ||
67 | await servers[1].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 4 ] }) | ||
68 | await waitJobs(servers) | ||
69 | await processViewsBuffer(servers) | ||
70 | |||
71 | await checkCounter('views', videoUUID, 2) | ||
72 | }) | ||
73 | }) | ||
74 | |||
75 | describe('Test views and viewers counters on live and VOD', function () { | ||
76 | let liveVideoId: string | ||
77 | let vodVideoId: string | ||
78 | let command: FfmpegCommand | ||
79 | |||
80 | before(async function () { | ||
81 | this.timeout(60000); | ||
82 | |||
83 | ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) | ||
84 | }) | ||
85 | |||
86 | it('Should display no views and viewers', async function () { | ||
87 | await checkCounter('views', liveVideoId, 0) | ||
88 | await checkCounter('viewers', liveVideoId, 0) | ||
89 | |||
90 | await checkCounter('views', vodVideoId, 0) | ||
91 | await checkCounter('viewers', vodVideoId, 0) | ||
92 | }) | ||
93 | |||
94 | it('Should view twice and display 1 view/viewer', async function () { | ||
95 | this.timeout(30000) | ||
96 | |||
97 | await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) | ||
98 | await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) | ||
99 | await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) | ||
100 | await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) | ||
101 | |||
102 | await waitJobs(servers) | ||
103 | await checkCounter('viewers', liveVideoId, 1) | ||
104 | await checkCounter('viewers', vodVideoId, 1) | ||
105 | |||
106 | await processViewsBuffer(servers) | ||
107 | |||
108 | await checkCounter('views', liveVideoId, 1) | ||
109 | await checkCounter('views', vodVideoId, 1) | ||
110 | }) | ||
111 | |||
112 | it('Should wait and display 0 viewers but still have 1 view', async function () { | ||
113 | this.timeout(30000) | ||
114 | |||
115 | await wait(12000) | ||
116 | await waitJobs(servers) | ||
117 | |||
118 | await checkCounter('views', liveVideoId, 1) | ||
119 | await checkCounter('viewers', liveVideoId, 0) | ||
120 | |||
121 | await checkCounter('views', vodVideoId, 1) | ||
122 | await checkCounter('viewers', vodVideoId, 0) | ||
123 | }) | ||
124 | |||
125 | it('Should view on a remote and on local and display 2 viewers and 3 views', async function () { | ||
126 | this.timeout(30000) | ||
127 | |||
128 | await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) | ||
129 | await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) | ||
130 | await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) | ||
131 | |||
132 | await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) | ||
133 | await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) | ||
134 | await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) | ||
135 | |||
136 | await waitJobs(servers) | ||
137 | |||
138 | await checkCounter('viewers', liveVideoId, 2) | ||
139 | await checkCounter('viewers', vodVideoId, 2) | ||
140 | |||
141 | await processViewsBuffer(servers) | ||
142 | |||
143 | await checkCounter('views', liveVideoId, 3) | ||
144 | await checkCounter('views', vodVideoId, 3) | ||
145 | }) | ||
146 | |||
147 | after(async function () { | ||
148 | await stopFfmpeg(command) | ||
149 | }) | ||
150 | }) | ||
151 | |||
152 | after(async function () { | ||
153 | await cleanupTests(servers) | ||
154 | }) | ||
155 | }) | ||
diff --git a/server/tests/api/views/video-views-overall-stats.ts b/server/tests/api/views/video-views-overall-stats.ts new file mode 100644 index 000000000..22761d6ec --- /dev/null +++ b/server/tests/api/views/video-views-overall-stats.ts | |||
@@ -0,0 +1,291 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import 'mocha' | ||
4 | import * as chai from 'chai' | ||
5 | import { FfmpegCommand } from 'fluent-ffmpeg' | ||
6 | import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@server/tests/shared' | ||
7 | import { cleanupTests, PeerTubeServer, stopFfmpeg, waitJobs } from '@shared/server-commands' | ||
8 | |||
9 | const expect = chai.expect | ||
10 | |||
11 | describe('Test views overall stats', function () { | ||
12 | let servers: PeerTubeServer[] | ||
13 | |||
14 | before(async function () { | ||
15 | this.timeout(120000) | ||
16 | |||
17 | servers = await prepareViewsServers() | ||
18 | }) | ||
19 | |||
20 | describe('Test rates and comments of local videos on VOD', function () { | ||
21 | let vodVideoId: string | ||
22 | |||
23 | before(async function () { | ||
24 | this.timeout(60000); | ||
25 | |||
26 | ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true })) | ||
27 | }) | ||
28 | |||
29 | it('Should have the appropriate likes', async function () { | ||
30 | this.timeout(60000) | ||
31 | |||
32 | await servers[0].videos.rate({ id: vodVideoId, rating: 'like' }) | ||
33 | await servers[1].videos.rate({ id: vodVideoId, rating: 'like' }) | ||
34 | |||
35 | await waitJobs(servers) | ||
36 | |||
37 | const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) | ||
38 | |||
39 | expect(stats.likes).to.equal(2) | ||
40 | expect(stats.dislikes).to.equal(0) | ||
41 | }) | ||
42 | |||
43 | it('Should have the appropriate dislikes', async function () { | ||
44 | this.timeout(60000) | ||
45 | |||
46 | await servers[0].videos.rate({ id: vodVideoId, rating: 'dislike' }) | ||
47 | await servers[1].videos.rate({ id: vodVideoId, rating: 'dislike' }) | ||
48 | |||
49 | await waitJobs(servers) | ||
50 | |||
51 | const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) | ||
52 | |||
53 | expect(stats.likes).to.equal(0) | ||
54 | expect(stats.dislikes).to.equal(2) | ||
55 | }) | ||
56 | |||
57 | it('Should have the appropriate comments', async function () { | ||
58 | this.timeout(60000) | ||
59 | |||
60 | await servers[0].comments.createThread({ videoId: vodVideoId, text: 'root' }) | ||
61 | await servers[0].comments.addReplyToLastThread({ text: 'reply' }) | ||
62 | await servers[1].comments.createThread({ videoId: vodVideoId, text: 'root' }) | ||
63 | |||
64 | await waitJobs(servers) | ||
65 | |||
66 | const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) | ||
67 | expect(stats.comments).to.equal(3) | ||
68 | }) | ||
69 | }) | ||
70 | |||
71 | describe('Test watch time stats of local videos on live and VOD', function () { | ||
72 | let vodVideoId: string | ||
73 | let liveVideoId: string | ||
74 | let command: FfmpegCommand | ||
75 | |||
76 | before(async function () { | ||
77 | this.timeout(60000); | ||
78 | |||
79 | ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) | ||
80 | }) | ||
81 | |||
82 | it('Should display overall stats of a video with no viewers', async function () { | ||
83 | for (const videoId of [ liveVideoId, vodVideoId ]) { | ||
84 | const stats = await servers[0].videoStats.getOverallStats({ videoId }) | ||
85 | |||
86 | expect(stats.views).to.equal(0) | ||
87 | expect(stats.averageWatchTime).to.equal(0) | ||
88 | expect(stats.totalWatchTime).to.equal(0) | ||
89 | } | ||
90 | }) | ||
91 | |||
92 | it('Should display overall stats with 1 viewer below the watch time limit', async function () { | ||
93 | this.timeout(60000) | ||
94 | |||
95 | for (const videoId of [ liveVideoId, vodVideoId ]) { | ||
96 | await servers[0].views.simulateViewer({ id: videoId, currentTimes: [ 0, 1 ] }) | ||
97 | } | ||
98 | |||
99 | await processViewersStats(servers) | ||
100 | |||
101 | for (const videoId of [ liveVideoId, vodVideoId ]) { | ||
102 | const stats = await servers[0].videoStats.getOverallStats({ videoId }) | ||
103 | |||
104 | expect(stats.views).to.equal(0) | ||
105 | expect(stats.averageWatchTime).to.equal(1) | ||
106 | expect(stats.totalWatchTime).to.equal(1) | ||
107 | } | ||
108 | }) | ||
109 | |||
110 | it('Should display overall stats with 2 viewers', async function () { | ||
111 | this.timeout(60000) | ||
112 | |||
113 | { | ||
114 | await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 3 ] }) | ||
115 | await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35, 40 ] }) | ||
116 | |||
117 | await processViewersStats(servers) | ||
118 | |||
119 | { | ||
120 | const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) | ||
121 | expect(stats.views).to.equal(1) | ||
122 | expect(stats.averageWatchTime).to.equal(2) | ||
123 | expect(stats.totalWatchTime).to.equal(4) | ||
124 | } | ||
125 | |||
126 | { | ||
127 | const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId }) | ||
128 | expect(stats.views).to.equal(1) | ||
129 | expect(stats.averageWatchTime).to.equal(21) | ||
130 | expect(stats.totalWatchTime).to.equal(41) | ||
131 | } | ||
132 | } | ||
133 | }) | ||
134 | |||
135 | it('Should display overall stats with a remote viewer below the watch time limit', async function () { | ||
136 | this.timeout(60000) | ||
137 | |||
138 | for (const videoId of [ liveVideoId, vodVideoId ]) { | ||
139 | await servers[1].views.simulateViewer({ id: videoId, currentTimes: [ 0, 2 ] }) | ||
140 | } | ||
141 | |||
142 | await processViewersStats(servers) | ||
143 | |||
144 | { | ||
145 | const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) | ||
146 | |||
147 | expect(stats.views).to.equal(1) | ||
148 | expect(stats.averageWatchTime).to.equal(2) | ||
149 | expect(stats.totalWatchTime).to.equal(6) | ||
150 | } | ||
151 | |||
152 | { | ||
153 | const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId }) | ||
154 | |||
155 | expect(stats.views).to.equal(1) | ||
156 | expect(stats.averageWatchTime).to.equal(14) | ||
157 | expect(stats.totalWatchTime).to.equal(43) | ||
158 | } | ||
159 | }) | ||
160 | |||
161 | it('Should display overall stats with a remote viewer above the watch time limit', async function () { | ||
162 | this.timeout(60000) | ||
163 | |||
164 | await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) | ||
165 | await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 45 ] }) | ||
166 | await processViewersStats(servers) | ||
167 | |||
168 | { | ||
169 | const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) | ||
170 | expect(stats.views).to.equal(2) | ||
171 | expect(stats.averageWatchTime).to.equal(3) | ||
172 | expect(stats.totalWatchTime).to.equal(11) | ||
173 | } | ||
174 | |||
175 | { | ||
176 | const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId }) | ||
177 | expect(stats.views).to.equal(2) | ||
178 | expect(stats.averageWatchTime).to.equal(22) | ||
179 | expect(stats.totalWatchTime).to.equal(88) | ||
180 | } | ||
181 | }) | ||
182 | |||
183 | after(async function () { | ||
184 | await stopFfmpeg(command) | ||
185 | }) | ||
186 | }) | ||
187 | |||
188 | describe('Test watchers peak stats of local videos on VOD', function () { | ||
189 | let videoUUID: string | ||
190 | |||
191 | before(async function () { | ||
192 | this.timeout(60000); | ||
193 | |||
194 | ({ vodVideoId: videoUUID } = await prepareViewsVideos({ servers, live: true, vod: true })) | ||
195 | }) | ||
196 | |||
197 | it('Should not have watchers peak', async function () { | ||
198 | const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID }) | ||
199 | |||
200 | expect(stats.viewersPeak).to.equal(0) | ||
201 | expect(stats.viewersPeakDate).to.be.null | ||
202 | }) | ||
203 | |||
204 | it('Should have watcher peak with 1 watcher', async function () { | ||
205 | this.timeout(60000) | ||
206 | |||
207 | const before = new Date() | ||
208 | await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 0, 2 ] }) | ||
209 | const after = new Date() | ||
210 | |||
211 | await processViewersStats(servers) | ||
212 | |||
213 | const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID }) | ||
214 | |||
215 | expect(stats.viewersPeak).to.equal(1) | ||
216 | expect(new Date(stats.viewersPeakDate)).to.be.above(before).and.below(after) | ||
217 | }) | ||
218 | |||
219 | it('Should have watcher peak with 2 watchers', async function () { | ||
220 | this.timeout(60000) | ||
221 | |||
222 | const before = new Date() | ||
223 | await servers[0].views.view({ id: videoUUID, currentTime: 0 }) | ||
224 | await servers[1].views.view({ id: videoUUID, currentTime: 0 }) | ||
225 | await servers[0].views.view({ id: videoUUID, currentTime: 2 }) | ||
226 | await servers[1].views.view({ id: videoUUID, currentTime: 2 }) | ||
227 | const after = new Date() | ||
228 | |||
229 | await processViewersStats(servers) | ||
230 | |||
231 | const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID }) | ||
232 | |||
233 | expect(stats.viewersPeak).to.equal(2) | ||
234 | expect(new Date(stats.viewersPeakDate)).to.be.above(before).and.below(after) | ||
235 | }) | ||
236 | }) | ||
237 | |||
238 | describe('Test countries', function () { | ||
239 | |||
240 | it('Should not report countries if geoip is disabled', async function () { | ||
241 | this.timeout(60000) | ||
242 | |||
243 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) | ||
244 | await waitJobs(servers) | ||
245 | |||
246 | await servers[1].views.view({ id: uuid, xForwardedFor: '8.8.8.8,127.0.0.1', currentTime: 1 }) | ||
247 | |||
248 | await processViewersStats(servers) | ||
249 | |||
250 | const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid }) | ||
251 | expect(stats.countries).to.have.lengthOf(0) | ||
252 | }) | ||
253 | |||
254 | it('Should report countries if geoip is enabled', async function () { | ||
255 | this.timeout(60000) | ||
256 | |||
257 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) | ||
258 | await waitJobs(servers) | ||
259 | |||
260 | await Promise.all([ | ||
261 | servers[0].kill(), | ||
262 | servers[1].kill() | ||
263 | ]) | ||
264 | |||
265 | const config = { geo_ip: { enabled: true } } | ||
266 | await Promise.all([ | ||
267 | servers[0].run(config), | ||
268 | servers[1].run(config) | ||
269 | ]) | ||
270 | |||
271 | await servers[0].views.view({ id: uuid, xForwardedFor: '8.8.8.8,127.0.0.1', currentTime: 1 }) | ||
272 | await servers[1].views.view({ id: uuid, xForwardedFor: '8.8.8.4,127.0.0.1', currentTime: 3 }) | ||
273 | await servers[1].views.view({ id: uuid, xForwardedFor: '80.67.169.12,127.0.0.1', currentTime: 2 }) | ||
274 | |||
275 | await processViewersStats(servers) | ||
276 | |||
277 | const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid }) | ||
278 | expect(stats.countries).to.have.lengthOf(2) | ||
279 | |||
280 | expect(stats.countries[0].isoCode).to.equal('US') | ||
281 | expect(stats.countries[0].viewers).to.equal(2) | ||
282 | |||
283 | expect(stats.countries[1].isoCode).to.equal('FR') | ||
284 | expect(stats.countries[1].viewers).to.equal(1) | ||
285 | }) | ||
286 | }) | ||
287 | |||
288 | after(async function () { | ||
289 | await cleanupTests(servers) | ||
290 | }) | ||
291 | }) | ||
diff --git a/server/tests/api/views/video-views-retention-stats.ts b/server/tests/api/views/video-views-retention-stats.ts new file mode 100644 index 000000000..98be7bfdb --- /dev/null +++ b/server/tests/api/views/video-views-retention-stats.ts | |||
@@ -0,0 +1,56 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import 'mocha' | ||
4 | import * as chai from 'chai' | ||
5 | import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@server/tests/shared' | ||
6 | import { cleanupTests, PeerTubeServer } from '@shared/server-commands' | ||
7 | |||
8 | const expect = chai.expect | ||
9 | |||
10 | describe('Test views retention stats', function () { | ||
11 | let servers: PeerTubeServer[] | ||
12 | |||
13 | before(async function () { | ||
14 | this.timeout(120000) | ||
15 | |||
16 | servers = await prepareViewsServers() | ||
17 | }) | ||
18 | |||
19 | describe('Test retention stats on VOD', function () { | ||
20 | let vodVideoId: string | ||
21 | |||
22 | before(async function () { | ||
23 | this.timeout(60000); | ||
24 | |||
25 | ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true })) | ||
26 | }) | ||
27 | |||
28 | it('Should display empty retention', async function () { | ||
29 | const { data } = await servers[0].videoStats.getRetentionStats({ videoId: vodVideoId }) | ||
30 | expect(data).to.have.lengthOf(6) | ||
31 | |||
32 | for (let i = 0; i < 6; i++) { | ||
33 | expect(data[i].second).to.equal(i) | ||
34 | expect(data[i].retentionPercent).to.equal(0) | ||
35 | } | ||
36 | }) | ||
37 | |||
38 | it('Should display appropriate retention metrics', async function () { | ||
39 | await servers[0].views.simulateViewer({ xForwardedFor: '127.0.0.2,127.0.0.1', id: vodVideoId, currentTimes: [ 0, 1 ] }) | ||
40 | await servers[0].views.simulateViewer({ xForwardedFor: '127.0.0.3,127.0.0.1', id: vodVideoId, currentTimes: [ 1, 3 ] }) | ||
41 | await servers[1].views.simulateViewer({ xForwardedFor: '127.0.0.2,127.0.0.1', id: vodVideoId, currentTimes: [ 4 ] }) | ||
42 | await servers[1].views.simulateViewer({ xForwardedFor: '127.0.0.3,127.0.0.1', id: vodVideoId, currentTimes: [ 0, 1 ] }) | ||
43 | |||
44 | await processViewersStats(servers) | ||
45 | |||
46 | const { data } = await servers[0].videoStats.getRetentionStats({ videoId: vodVideoId }) | ||
47 | expect(data).to.have.lengthOf(6) | ||
48 | |||
49 | expect(data.map(d => d.retentionPercent)).to.deep.equal([ 50, 75, 25, 25, 25, 0 ]) | ||
50 | }) | ||
51 | }) | ||
52 | |||
53 | after(async function () { | ||
54 | await cleanupTests(servers) | ||
55 | }) | ||
56 | }) | ||
diff --git a/server/tests/api/views/video-views-timeserie-stats.ts b/server/tests/api/views/video-views-timeserie-stats.ts new file mode 100644 index 000000000..98c041cdf --- /dev/null +++ b/server/tests/api/views/video-views-timeserie-stats.ts | |||
@@ -0,0 +1,109 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import 'mocha' | ||
4 | import * as chai from 'chai' | ||
5 | import { FfmpegCommand } from 'fluent-ffmpeg' | ||
6 | import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@server/tests/shared' | ||
7 | import { VideoStatsTimeserie, VideoStatsTimeserieMetric } from '@shared/models' | ||
8 | import { cleanupTests, PeerTubeServer, stopFfmpeg } from '@shared/server-commands' | ||
9 | |||
10 | const expect = chai.expect | ||
11 | |||
12 | describe('Test views timeserie stats', function () { | ||
13 | const availableMetrics: VideoStatsTimeserieMetric[] = [ 'viewers' ] | ||
14 | |||
15 | let servers: PeerTubeServer[] | ||
16 | |||
17 | before(async function () { | ||
18 | this.timeout(120000) | ||
19 | |||
20 | servers = await prepareViewsServers() | ||
21 | }) | ||
22 | |||
23 | describe('Common metric tests', function () { | ||
24 | let vodVideoId: string | ||
25 | |||
26 | before(async function () { | ||
27 | this.timeout(60000); | ||
28 | |||
29 | ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true })) | ||
30 | }) | ||
31 | |||
32 | it('Should display empty metric stats', async function () { | ||
33 | for (const metric of availableMetrics) { | ||
34 | const { data } = await servers[0].videoStats.getTimeserieStats({ videoId: vodVideoId, metric }) | ||
35 | |||
36 | expect(data).to.have.lengthOf(30) | ||
37 | |||
38 | for (const d of data) { | ||
39 | expect(d.value).to.equal(0) | ||
40 | } | ||
41 | } | ||
42 | }) | ||
43 | }) | ||
44 | |||
45 | describe('Test viewer and watch time metrics on live and VOD', function () { | ||
46 | let vodVideoId: string | ||
47 | let liveVideoId: string | ||
48 | let command: FfmpegCommand | ||
49 | |||
50 | function expectTimeserieData (result: VideoStatsTimeserie, lastValue: number) { | ||
51 | const { data } = result | ||
52 | expect(data).to.have.lengthOf(30) | ||
53 | |||
54 | const last = data[data.length - 1] | ||
55 | |||
56 | const today = new Date().getDate() | ||
57 | expect(new Date(last.date).getDate()).to.equal(today) | ||
58 | expect(last.value).to.equal(lastValue) | ||
59 | |||
60 | for (let i = 0; i < data.length - 2; i++) { | ||
61 | expect(data[i].value).to.equal(0) | ||
62 | } | ||
63 | } | ||
64 | |||
65 | before(async function () { | ||
66 | this.timeout(60000); | ||
67 | |||
68 | ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) | ||
69 | }) | ||
70 | |||
71 | it('Should display appropriate viewers metrics', async function () { | ||
72 | for (const videoId of [ vodVideoId, liveVideoId ]) { | ||
73 | await servers[0].views.simulateViewer({ id: videoId, currentTimes: [ 0, 3 ] }) | ||
74 | await servers[1].views.simulateViewer({ id: videoId, currentTimes: [ 0, 5 ] }) | ||
75 | } | ||
76 | |||
77 | await processViewersStats(servers) | ||
78 | |||
79 | for (const videoId of [ vodVideoId, liveVideoId ]) { | ||
80 | const result = await servers[0].videoStats.getTimeserieStats({ videoId, metric: 'viewers' }) | ||
81 | expectTimeserieData(result, 2) | ||
82 | } | ||
83 | }) | ||
84 | |||
85 | it('Should display appropriate watch time metrics', async function () { | ||
86 | for (const videoId of [ vodVideoId, liveVideoId ]) { | ||
87 | const result = await servers[0].videoStats.getTimeserieStats({ videoId, metric: 'aggregateWatchTime' }) | ||
88 | expectTimeserieData(result, 8) | ||
89 | |||
90 | await servers[1].views.simulateViewer({ id: videoId, currentTimes: [ 0, 1 ] }) | ||
91 | } | ||
92 | |||
93 | await processViewersStats(servers) | ||
94 | |||
95 | for (const videoId of [ vodVideoId, liveVideoId ]) { | ||
96 | const result = await servers[0].videoStats.getTimeserieStats({ videoId, metric: 'aggregateWatchTime' }) | ||
97 | expectTimeserieData(result, 9) | ||
98 | } | ||
99 | }) | ||
100 | |||
101 | after(async function () { | ||
102 | await stopFfmpeg(command) | ||
103 | }) | ||
104 | }) | ||
105 | |||
106 | after(async function () { | ||
107 | await cleanupTests(servers) | ||
108 | }) | ||
109 | }) | ||
diff --git a/server/tests/api/videos/videos-views-cleaner.ts b/server/tests/api/views/videos-views-cleaner.ts index e6815a4a8..ef988837f 100644 --- a/server/tests/api/videos/videos-views-cleaner.ts +++ b/server/tests/api/views/videos-views-cleaner.ts | |||
@@ -34,10 +34,10 @@ describe('Test video views cleaner', function () { | |||
34 | 34 | ||
35 | await waitJobs(servers) | 35 | await waitJobs(servers) |
36 | 36 | ||
37 | await servers[0].videos.view({ id: videoIdServer1 }) | 37 | await servers[0].views.simulateView({ id: videoIdServer1 }) |
38 | await servers[1].videos.view({ id: videoIdServer1 }) | 38 | await servers[1].views.simulateView({ id: videoIdServer1 }) |
39 | await servers[0].videos.view({ id: videoIdServer2 }) | 39 | await servers[0].views.simulateView({ id: videoIdServer2 }) |
40 | await servers[1].videos.view({ id: videoIdServer2 }) | 40 | await servers[1].views.simulateView({ id: videoIdServer2 }) |
41 | 41 | ||
42 | await waitJobs(servers) | 42 | await waitJobs(servers) |
43 | }) | 43 | }) |
diff --git a/server/tests/plugins/action-hooks.ts b/server/tests/plugins/action-hooks.ts index 8788a9644..57ede2701 100644 --- a/server/tests/plugins/action-hooks.ts +++ b/server/tests/plugins/action-hooks.ts | |||
@@ -1,6 +1,7 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | 1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | 2 | ||
3 | import 'mocha' | 3 | import 'mocha' |
4 | import { ServerHookName, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models' | ||
4 | import { | 5 | import { |
5 | cleanupTests, | 6 | cleanupTests, |
6 | createMultipleServers, | 7 | createMultipleServers, |
@@ -10,7 +11,6 @@ import { | |||
10 | setAccessTokensToServers, | 11 | setAccessTokensToServers, |
11 | setDefaultVideoChannel | 12 | setDefaultVideoChannel |
12 | } from '@shared/server-commands' | 13 | } from '@shared/server-commands' |
13 | import { ServerHookName, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models' | ||
14 | 14 | ||
15 | describe('Test plugin action hooks', function () { | 15 | describe('Test plugin action hooks', function () { |
16 | let servers: PeerTubeServer[] | 16 | let servers: PeerTubeServer[] |
@@ -61,7 +61,7 @@ describe('Test plugin action hooks', function () { | |||
61 | }) | 61 | }) |
62 | 62 | ||
63 | it('Should run action:api.video.viewed', async function () { | 63 | it('Should run action:api.video.viewed', async function () { |
64 | await servers[0].videos.view({ id: videoUUID }) | 64 | await servers[0].views.simulateView({ id: videoUUID }) |
65 | 65 | ||
66 | await checkHook('action:api.video.viewed') | 66 | await checkHook('action:api.video.viewed') |
67 | }) | 67 | }) |
diff --git a/server/tests/plugins/plugin-helpers.ts b/server/tests/plugins/plugin-helpers.ts index 167429ef4..5e8d08dff 100644 --- a/server/tests/plugins/plugin-helpers.ts +++ b/server/tests/plugins/plugin-helpers.ts | |||
@@ -301,7 +301,7 @@ describe('Test plugin helpers', function () { | |||
301 | // Should not throw -> video exists | 301 | // Should not throw -> video exists |
302 | const video = await servers[0].videos.get({ id: videoUUID }) | 302 | const video = await servers[0].videos.get({ id: videoUUID }) |
303 | // Should delete the video | 303 | // Should delete the video |
304 | await servers[0].videos.view({ id: videoUUID }) | 304 | await servers[0].views.simulateView({ id: videoUUID }) |
305 | 305 | ||
306 | await servers[0].servers.waitUntilLog('Video deleted by plugin four.') | 306 | await servers[0].servers.waitUntilLog('Video deleted by plugin four.') |
307 | 307 | ||
diff --git a/server/tests/shared/index.ts b/server/tests/shared/index.ts index 47019d6a8..9f7ade53d 100644 --- a/server/tests/shared/index.ts +++ b/server/tests/shared/index.ts | |||
@@ -13,3 +13,4 @@ export * from './streaming-playlists' | |||
13 | export * from './tests' | 13 | export * from './tests' |
14 | export * from './tracker' | 14 | export * from './tracker' |
15 | export * from './videos' | 15 | export * from './videos' |
16 | export * from './views' | ||
diff --git a/server/tests/shared/views.ts b/server/tests/shared/views.ts new file mode 100644 index 000000000..e6b289715 --- /dev/null +++ b/server/tests/shared/views.ts | |||
@@ -0,0 +1,93 @@ | |||
1 | import { FfmpegCommand } from 'fluent-ffmpeg' | ||
2 | import { wait } from '@shared/core-utils' | ||
3 | import { VideoCreateResult, VideoPrivacy } from '@shared/models' | ||
4 | import { | ||
5 | createMultipleServers, | ||
6 | doubleFollow, | ||
7 | PeerTubeServer, | ||
8 | setAccessTokensToServers, | ||
9 | setDefaultVideoChannel, | ||
10 | waitJobs, | ||
11 | waitUntilLivePublishedOnAllServers | ||
12 | } from '@shared/server-commands' | ||
13 | |||
14 | async function processViewersStats (servers: PeerTubeServer[]) { | ||
15 | await wait(6000) | ||
16 | |||
17 | for (const server of servers) { | ||
18 | await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } }) | ||
19 | await server.debug.sendCommand({ body: { command: 'process-video-viewers' } }) | ||
20 | } | ||
21 | |||
22 | await waitJobs(servers) | ||
23 | } | ||
24 | |||
25 | async function processViewsBuffer (servers: PeerTubeServer[]) { | ||
26 | for (const server of servers) { | ||
27 | await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } }) | ||
28 | } | ||
29 | |||
30 | await waitJobs(servers) | ||
31 | } | ||
32 | |||
33 | async function prepareViewsServers () { | ||
34 | const servers = await createMultipleServers(2) | ||
35 | await setAccessTokensToServers(servers) | ||
36 | await setDefaultVideoChannel(servers) | ||
37 | |||
38 | await servers[0].config.updateCustomSubConfig({ | ||
39 | newConfig: { | ||
40 | live: { | ||
41 | enabled: true, | ||
42 | allowReplay: true, | ||
43 | transcoding: { | ||
44 | enabled: false | ||
45 | } | ||
46 | } | ||
47 | } | ||
48 | }) | ||
49 | |||
50 | await doubleFollow(servers[0], servers[1]) | ||
51 | |||
52 | return servers | ||
53 | } | ||
54 | |||
55 | async function prepareViewsVideos (options: { | ||
56 | servers: PeerTubeServer[] | ||
57 | live: boolean | ||
58 | vod: boolean | ||
59 | }) { | ||
60 | const { servers } = options | ||
61 | |||
62 | const liveAttributes = { | ||
63 | name: 'live video', | ||
64 | channelId: servers[0].store.channel.id, | ||
65 | privacy: VideoPrivacy.PUBLIC | ||
66 | } | ||
67 | |||
68 | let ffmpegCommand: FfmpegCommand | ||
69 | let live: VideoCreateResult | ||
70 | let vod: VideoCreateResult | ||
71 | |||
72 | if (options.live) { | ||
73 | live = await servers[0].live.create({ fields: liveAttributes }) | ||
74 | |||
75 | ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: live.uuid }) | ||
76 | await waitUntilLivePublishedOnAllServers(servers, live.uuid) | ||
77 | } | ||
78 | |||
79 | if (options.vod) { | ||
80 | vod = await servers[0].videos.quickUpload({ name: 'video' }) | ||
81 | } | ||
82 | |||
83 | await waitJobs(servers) | ||
84 | |||
85 | return { liveVideoId: live?.uuid, vodVideoId: vod?.uuid, ffmpegCommand } | ||
86 | } | ||
87 | |||
88 | export { | ||
89 | processViewersStats, | ||
90 | prepareViewsServers, | ||
91 | processViewsBuffer, | ||
92 | prepareViewsVideos | ||
93 | } | ||
diff --git a/server/types/express.d.ts b/server/types/express.d.ts index 91a8cf3d8..4537c57c6 100644 --- a/server/types/express.d.ts +++ b/server/types/express.d.ts | |||
@@ -185,6 +185,8 @@ declare module 'express' { | |||
185 | externalAuth?: RegisterServerAuthExternalOptions | 185 | externalAuth?: RegisterServerAuthExternalOptions |
186 | 186 | ||
187 | plugin?: MPlugin | 187 | plugin?: MPlugin |
188 | |||
189 | localViewerFull?: MLocalVideoViewerWithWatchSections | ||
188 | } | 190 | } |
189 | } | 191 | } |
190 | } | 192 | } |
diff --git a/server/types/models/video/index.ts b/server/types/models/video/index.ts index e586a4e42..5ddffcab5 100644 --- a/server/types/models/video/index.ts +++ b/server/types/models/video/index.ts | |||
@@ -1,3 +1,5 @@ | |||
1 | export * from './local-video-viewer-watch-section' | ||
2 | export * from './local-video-viewer' | ||
1 | export * from './schedule-video-update' | 3 | export * from './schedule-video-update' |
2 | export * from './tag' | 4 | export * from './tag' |
3 | export * from './thumbnail' | 5 | export * from './thumbnail' |
diff --git a/server/types/models/video/local-video-viewer-watch-section.ts b/server/types/models/video/local-video-viewer-watch-section.ts new file mode 100644 index 000000000..be7a3bba0 --- /dev/null +++ b/server/types/models/video/local-video-viewer-watch-section.ts | |||
@@ -0,0 +1,5 @@ | |||
1 | import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' | ||
2 | |||
3 | // ############################################################################ | ||
4 | |||
5 | export type MLocalVideoViewerWatchSection = Omit<LocalVideoViewerWatchSectionModel, 'LocalVideoViewerModel'> | ||
diff --git a/server/types/models/video/local-video-viewer.ts b/server/types/models/video/local-video-viewer.ts new file mode 100644 index 000000000..b78ef5507 --- /dev/null +++ b/server/types/models/video/local-video-viewer.ts | |||
@@ -0,0 +1,19 @@ | |||
1 | import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' | ||
2 | import { PickWith } from '@shared/typescript-utils' | ||
3 | import { MLocalVideoViewerWatchSection } from './local-video-viewer-watch-section' | ||
4 | import { MVideo } from './video' | ||
5 | |||
6 | type Use<K extends keyof LocalVideoViewerModel, M> = PickWith<LocalVideoViewerModel, K, M> | ||
7 | |||
8 | // ############################################################################ | ||
9 | |||
10 | export type MLocalVideoViewer = Omit<LocalVideoViewerModel, 'Video'> | ||
11 | |||
12 | export type MLocalVideoViewerVideo = | ||
13 | MLocalVideoViewer & | ||
14 | Use<'Video', MVideo> | ||
15 | |||
16 | export type MLocalVideoViewerWithWatchSections = | ||
17 | MLocalVideoViewer & | ||
18 | Use<'Video', MVideo> & | ||
19 | Use<'WatchSections', MLocalVideoViewerWatchSection[]> | ||