diff options
author | Chocobozzz <me@florianbigard.com> | 2019-03-19 14:13:53 +0100 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2019-03-19 14:13:53 +0100 |
commit | 9f79ade627f0044606a9fbbe16ca0154661d12b9 (patch) | |
tree | ccfc1876da591bc52cbb4b8ccddedbeae7876679 | |
parent | 0e0c745b62b2f18e228328e8c4f95b17e54f7a5e (diff) | |
download | PeerTube-9f79ade627f0044606a9fbbe16ca0154661d12b9.tar.gz PeerTube-9f79ade627f0044606a9fbbe16ca0154661d12b9.tar.zst PeerTube-9f79ade627f0044606a9fbbe16ca0154661d12b9.zip |
Refresh playlists
-rw-r--r-- | server/controllers/api/video-playlist.ts | 6 | ||||
-rw-r--r-- | server/initializers/constants.ts | 4 | ||||
-rw-r--r-- | server/lib/activitypub/actor.ts | 3 | ||||
-rw-r--r-- | server/lib/activitypub/playlist.ts | 56 | ||||
-rw-r--r-- | server/lib/job-queue/handlers/activitypub-refresher.ts | 14 | ||||
-rw-r--r-- | server/middlewares/validators/videos/video-playlists.ts | 3 | ||||
-rw-r--r-- | server/models/activitypub/actor.ts | 9 | ||||
-rw-r--r-- | server/models/utils.ts | 12 | ||||
-rw-r--r-- | server/models/video/video-playlist.ts | 15 | ||||
-rw-r--r-- | server/models/video/video.ts | 11 | ||||
-rw-r--r-- | server/tests/api/activitypub/refresher.ts | 130 | ||||
-rw-r--r-- | shared/utils/miscs/sql.ts | 9 | ||||
-rw-r--r-- | shared/utils/videos/video-playlists.ts | 2 |
13 files changed, 215 insertions, 59 deletions
diff --git a/server/controllers/api/video-playlist.ts b/server/controllers/api/video-playlist.ts index 5b1601c4e..feba30564 100644 --- a/server/controllers/api/video-playlist.ts +++ b/server/controllers/api/video-playlist.ts | |||
@@ -40,6 +40,7 @@ import { VideoPlaylistElementUpdate } from '../../../shared/models/videos/playli | |||
40 | import { copy, pathExists } from 'fs-extra' | 40 | import { copy, pathExists } from 'fs-extra' |
41 | import { AccountModel } from '../../models/account/account' | 41 | import { AccountModel } from '../../models/account/account' |
42 | import { VideoPlaylistReorder } from '../../../shared/models/videos/playlist/video-playlist-reorder.model' | 42 | import { VideoPlaylistReorder } from '../../../shared/models/videos/playlist/video-playlist-reorder.model' |
43 | import { JobQueue } from '../../lib/job-queue' | ||
43 | 44 | ||
44 | const reqThumbnailFile = createReqFiles([ 'thumbnailfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, { thumbnailfile: CONFIG.STORAGE.TMP_DIR }) | 45 | const reqThumbnailFile = createReqFiles([ 'thumbnailfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, { thumbnailfile: CONFIG.STORAGE.TMP_DIR }) |
45 | 46 | ||
@@ -142,6 +143,11 @@ async function listVideoPlaylists (req: express.Request, res: express.Response) | |||
142 | function getVideoPlaylist (req: express.Request, res: express.Response) { | 143 | function getVideoPlaylist (req: express.Request, res: express.Response) { |
143 | const videoPlaylist = res.locals.videoPlaylist | 144 | const videoPlaylist = res.locals.videoPlaylist |
144 | 145 | ||
146 | if (videoPlaylist.isOutdated()) { | ||
147 | JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video-playlist', url: videoPlaylist.url } }) | ||
148 | .catch(err => logger.error('Cannot create AP refresher job for playlist %s.', videoPlaylist.url, { err })) | ||
149 | } | ||
150 | |||
145 | return res.json(videoPlaylist.toFormattedJSON()) | 151 | return res.json(videoPlaylist.toFormattedJSON()) |
146 | } | 152 | } |
147 | 153 | ||
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 19e63d6fe..7fac8a4d6 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -584,7 +584,8 @@ const ACTIVITY_PUB = { | |||
584 | }, | 584 | }, |
585 | MAX_RECURSION_COMMENTS: 100, | 585 | MAX_RECURSION_COMMENTS: 100, |
586 | ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days | 586 | ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days |
587 | VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2 // 2 days | 587 | VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days |
588 | VIDEO_PLAYLIST_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2 // 2 days | ||
588 | } | 589 | } |
589 | 590 | ||
590 | const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = { | 591 | const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = { |
@@ -724,6 +725,7 @@ if (isTestInstance() === true) { | |||
724 | ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2 | 725 | ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2 |
725 | ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 seconds | 726 | ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 seconds |
726 | ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL = 10 * 1000 // 10 seconds | 727 | ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL = 10 * 1000 // 10 seconds |
728 | ACTIVITY_PUB.VIDEO_PLAYLIST_REFRESH_INTERVAL = 10 * 1000 // 10 seconds | ||
727 | 729 | ||
728 | CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB | 730 | CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB |
729 | 731 | ||
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index f77df8b78..63e810642 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts | |||
@@ -375,7 +375,8 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe | |||
375 | } | 375 | } |
376 | 376 | ||
377 | if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) { | 377 | if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) { |
378 | throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id) | 378 | logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id) |
379 | return { result: undefined, statusCode: requestResult.response.statusCode } | ||
379 | } | 380 | } |
380 | 381 | ||
381 | const followersCount = await fetchActorTotalItems(actorJSON.followers) | 382 | const followersCount = await fetchActorTotalItems(actorJSON.followers) |
diff --git a/server/lib/activitypub/playlist.ts b/server/lib/activitypub/playlist.ts index 70389044e..c4a8f12ec 100644 --- a/server/lib/activitypub/playlist.ts +++ b/server/lib/activitypub/playlist.ts | |||
@@ -95,7 +95,7 @@ async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAc | |||
95 | return Promise.resolve() | 95 | return Promise.resolve() |
96 | }) | 96 | }) |
97 | 97 | ||
98 | // Empty playlists generally do not have a miniature, so skip it | 98 | // Empty playlists generally do not have a miniature, so skip this |
99 | if (accItems.length !== 0) { | 99 | if (accItems.length !== 0) { |
100 | try { | 100 | try { |
101 | await generateThumbnailFromUrl(playlist, playlistObject.icon) | 101 | await generateThumbnailFromUrl(playlist, playlistObject.icon) |
@@ -107,13 +107,45 @@ async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAc | |||
107 | return resetVideoPlaylistElements(accItems, playlist) | 107 | return resetVideoPlaylistElements(accItems, playlist) |
108 | } | 108 | } |
109 | 109 | ||
110 | async function refreshVideoPlaylistIfNeeded (videoPlaylist: VideoPlaylistModel): Promise<VideoPlaylistModel> { | ||
111 | if (!videoPlaylist.isOutdated()) return videoPlaylist | ||
112 | |||
113 | try { | ||
114 | const { statusCode, playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url) | ||
115 | if (statusCode === 404) { | ||
116 | logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url) | ||
117 | |||
118 | await videoPlaylist.destroy() | ||
119 | return undefined | ||
120 | } | ||
121 | |||
122 | if (playlistObject === undefined) { | ||
123 | logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url) | ||
124 | |||
125 | await videoPlaylist.setAsRefreshed() | ||
126 | return videoPlaylist | ||
127 | } | ||
128 | |||
129 | const byAccount = videoPlaylist.OwnerAccount | ||
130 | await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to) | ||
131 | |||
132 | return videoPlaylist | ||
133 | } catch (err) { | ||
134 | logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err }) | ||
135 | |||
136 | await videoPlaylist.setAsRefreshed() | ||
137 | return videoPlaylist | ||
138 | } | ||
139 | } | ||
140 | |||
110 | // --------------------------------------------------------------------------- | 141 | // --------------------------------------------------------------------------- |
111 | 142 | ||
112 | export { | 143 | export { |
113 | createAccountPlaylists, | 144 | createAccountPlaylists, |
114 | playlistObjectToDBAttributes, | 145 | playlistObjectToDBAttributes, |
115 | playlistElementObjectToDBAttributes, | 146 | playlistElementObjectToDBAttributes, |
116 | createOrUpdateVideoPlaylist | 147 | createOrUpdateVideoPlaylist, |
148 | refreshVideoPlaylistIfNeeded | ||
117 | } | 149 | } |
118 | 150 | ||
119 | // --------------------------------------------------------------------------- | 151 | // --------------------------------------------------------------------------- |
@@ -162,3 +194,23 @@ function generateThumbnailFromUrl (playlist: VideoPlaylistModel, icon: ActivityI | |||
162 | 194 | ||
163 | return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE) | 195 | return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE) |
164 | } | 196 | } |
197 | |||
198 | async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> { | ||
199 | const options = { | ||
200 | uri: playlistUrl, | ||
201 | method: 'GET', | ||
202 | json: true, | ||
203 | activityPub: true | ||
204 | } | ||
205 | |||
206 | logger.info('Fetching remote playlist %s.', playlistUrl) | ||
207 | |||
208 | const { response, body } = await doRequest(options) | ||
209 | |||
210 | if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) { | ||
211 | logger.debug('Remote video playlist JSON is not valid.', { body }) | ||
212 | return { statusCode: response.statusCode, playlistObject: undefined } | ||
213 | } | ||
214 | |||
215 | return { statusCode: response.statusCode, playlistObject: body } | ||
216 | } | ||
diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts index 454b975fe..4d6c38cfa 100644 --- a/server/lib/job-queue/handlers/activitypub-refresher.ts +++ b/server/lib/job-queue/handlers/activitypub-refresher.ts | |||
@@ -1,11 +1,12 @@ | |||
1 | import * as Bull from 'bull' | 1 | import * as Bull from 'bull' |
2 | import { logger } from '../../../helpers/logger' | 2 | import { logger } from '../../../helpers/logger' |
3 | import { fetchVideoByUrl } from '../../../helpers/video' | 3 | import { fetchVideoByUrl } from '../../../helpers/video' |
4 | import { refreshVideoIfNeeded, refreshActorIfNeeded } from '../../activitypub' | 4 | import { refreshActorIfNeeded, refreshVideoIfNeeded, refreshVideoPlaylistIfNeeded } from '../../activitypub' |
5 | import { ActorModel } from '../../../models/activitypub/actor' | 5 | import { ActorModel } from '../../../models/activitypub/actor' |
6 | import { VideoPlaylistModel } from '../../../models/video/video-playlist' | ||
6 | 7 | ||
7 | export type RefreshPayload = { | 8 | export type RefreshPayload = { |
8 | type: 'video' | 'actor' | 9 | type: 'video' | 'video-playlist' | 'actor' |
9 | url: string | 10 | url: string |
10 | } | 11 | } |
11 | 12 | ||
@@ -15,13 +16,13 @@ async function refreshAPObject (job: Bull.Job) { | |||
15 | logger.info('Processing AP refresher in job %d for %s.', job.id, payload.url) | 16 | logger.info('Processing AP refresher in job %d for %s.', job.id, payload.url) |
16 | 17 | ||
17 | if (payload.type === 'video') return refreshVideo(payload.url) | 18 | if (payload.type === 'video') return refreshVideo(payload.url) |
19 | if (payload.type === 'video-playlist') return refreshVideoPlaylist(payload.url) | ||
18 | if (payload.type === 'actor') return refreshActor(payload.url) | 20 | if (payload.type === 'actor') return refreshActor(payload.url) |
19 | } | 21 | } |
20 | 22 | ||
21 | // --------------------------------------------------------------------------- | 23 | // --------------------------------------------------------------------------- |
22 | 24 | ||
23 | export { | 25 | export { |
24 | refreshActor, | ||
25 | refreshAPObject | 26 | refreshAPObject |
26 | } | 27 | } |
27 | 28 | ||
@@ -50,5 +51,12 @@ async function refreshActor (actorUrl: string) { | |||
50 | if (actor) { | 51 | if (actor) { |
51 | await refreshActorIfNeeded(actor, fetchType) | 52 | await refreshActorIfNeeded(actor, fetchType) |
52 | } | 53 | } |
54 | } | ||
55 | |||
56 | async function refreshVideoPlaylist (playlistUrl: string) { | ||
57 | const playlist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(playlistUrl) | ||
53 | 58 | ||
59 | if (playlist) { | ||
60 | await refreshVideoPlaylistIfNeeded(playlist) | ||
61 | } | ||
54 | } | 62 | } |
diff --git a/server/middlewares/validators/videos/video-playlists.ts b/server/middlewares/validators/videos/video-playlists.ts index 55e09e354..6ba30fac9 100644 --- a/server/middlewares/validators/videos/video-playlists.ts +++ b/server/middlewares/validators/videos/video-playlists.ts | |||
@@ -8,8 +8,8 @@ import { doesVideoExist, isVideoImage } from '../../../helpers/custom-validators | |||
8 | import { CONSTRAINTS_FIELDS } from '../../../initializers' | 8 | import { CONSTRAINTS_FIELDS } from '../../../initializers' |
9 | import { isArrayOf, isIdOrUUIDValid, isIdValid, isUUIDValid, toIntArray, toValueOrNull } from '../../../helpers/custom-validators/misc' | 9 | import { isArrayOf, isIdOrUUIDValid, isIdValid, isUUIDValid, toIntArray, toValueOrNull } from '../../../helpers/custom-validators/misc' |
10 | import { | 10 | import { |
11 | isVideoPlaylistDescriptionValid, | ||
12 | doesVideoPlaylistExist, | 11 | doesVideoPlaylistExist, |
12 | isVideoPlaylistDescriptionValid, | ||
13 | isVideoPlaylistNameValid, | 13 | isVideoPlaylistNameValid, |
14 | isVideoPlaylistPrivacyValid, | 14 | isVideoPlaylistPrivacyValid, |
15 | isVideoPlaylistTimestampValid, | 15 | isVideoPlaylistTimestampValid, |
@@ -19,7 +19,6 @@ import { VideoPlaylistModel } from '../../../models/video/video-playlist' | |||
19 | import { cleanUpReqFiles } from '../../../helpers/express-utils' | 19 | import { cleanUpReqFiles } from '../../../helpers/express-utils' |
20 | import { doesVideoChannelIdExist } from '../../../helpers/custom-validators/video-channels' | 20 | import { doesVideoChannelIdExist } from '../../../helpers/custom-validators/video-channels' |
21 | import { VideoPlaylistElementModel } from '../../../models/video/video-playlist-element' | 21 | import { VideoPlaylistElementModel } from '../../../models/video/video-playlist-element' |
22 | import { VideoModel } from '../../../models/video/video' | ||
23 | import { authenticatePromiseIfNeeded } from '../../oauth' | 22 | import { authenticatePromiseIfNeeded } from '../../oauth' |
24 | import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model' | 23 | import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model' |
25 | import { VideoPlaylistType } from '../../../../shared/models/videos/playlist/video-playlist-type.model' | 24 | import { VideoPlaylistType } from '../../../../shared/models/videos/playlist/video-playlist-type.model' |
diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts index 2fceb21dd..7d91e8a4a 100644 --- a/server/models/activitypub/actor.ts +++ b/server/models/activitypub/actor.ts | |||
@@ -34,7 +34,7 @@ import { ACTIVITY_PUB, ACTIVITY_PUB_ACTOR_TYPES, CONFIG, CONSTRAINTS_FIELDS } fr | |||
34 | import { AccountModel } from '../account/account' | 34 | import { AccountModel } from '../account/account' |
35 | import { AvatarModel } from '../avatar/avatar' | 35 | import { AvatarModel } from '../avatar/avatar' |
36 | import { ServerModel } from '../server/server' | 36 | import { ServerModel } from '../server/server' |
37 | import { throwIfNotValid } from '../utils' | 37 | import { isOutdated, throwIfNotValid } from '../utils' |
38 | import { VideoChannelModel } from '../video/video-channel' | 38 | import { VideoChannelModel } from '../video/video-channel' |
39 | import { ActorFollowModel } from './actor-follow' | 39 | import { ActorFollowModel } from './actor-follow' |
40 | import { VideoModel } from '../video/video' | 40 | import { VideoModel } from '../video/video' |
@@ -532,11 +532,6 @@ export class ActorModel extends Model<ActorModel> { | |||
532 | isOutdated () { | 532 | isOutdated () { |
533 | if (this.isOwned()) return false | 533 | if (this.isOwned()) return false |
534 | 534 | ||
535 | const now = Date.now() | 535 | return isOutdated(this, ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL) |
536 | const createdAtTime = this.createdAt.getTime() | ||
537 | const updatedAtTime = this.updatedAt.getTime() | ||
538 | |||
539 | return (now - createdAtTime) > ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL && | ||
540 | (now - updatedAtTime) > ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL | ||
541 | } | 536 | } |
542 | } | 537 | } |
diff --git a/server/models/utils.ts b/server/models/utils.ts index 4ebd07dab..f8a71b270 100644 --- a/server/models/utils.ts +++ b/server/models/utils.ts | |||
@@ -1,5 +1,6 @@ | |||
1 | import { Sequelize } from 'sequelize-typescript' | 1 | import { Sequelize } from 'sequelize-typescript' |
2 | import * as validator from 'validator' | 2 | import * as validator from 'validator' |
3 | import { ACTIVITY_PUB } from '../initializers' | ||
3 | 4 | ||
4 | type SortType = { sortModel: any, sortValue: string } | 5 | type SortType = { sortModel: any, sortValue: string } |
5 | 6 | ||
@@ -44,6 +45,14 @@ function getSortOnModel (model: any, value: string, lastSort: string[] = [ 'id', | |||
44 | return [ firstSort, lastSort ] | 45 | return [ firstSort, lastSort ] |
45 | } | 46 | } |
46 | 47 | ||
48 | function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) { | ||
49 | const now = Date.now() | ||
50 | const createdAtTime = model.createdAt.getTime() | ||
51 | const updatedAtTime = model.updatedAt.getTime() | ||
52 | |||
53 | return (now - createdAtTime) > refreshInterval && (now - updatedAtTime) > refreshInterval | ||
54 | } | ||
55 | |||
47 | function throwIfNotValid (value: any, validator: (value: any) => boolean, fieldName = 'value') { | 56 | function throwIfNotValid (value: any, validator: (value: any) => boolean, fieldName = 'value') { |
48 | if (validator(value) === false) { | 57 | if (validator(value) === false) { |
49 | throw new Error(`"${value}" is not a valid ${fieldName}.`) | 58 | throw new Error(`"${value}" is not a valid ${fieldName}.`) |
@@ -108,7 +117,8 @@ export { | |||
108 | throwIfNotValid, | 117 | throwIfNotValid, |
109 | buildServerIdsFollowedBy, | 118 | buildServerIdsFollowedBy, |
110 | buildTrigramSearchIndex, | 119 | buildTrigramSearchIndex, |
111 | buildWhereIdOrUUID | 120 | buildWhereIdOrUUID, |
121 | isOutdated | ||
112 | } | 122 | } |
113 | 123 | ||
114 | // --------------------------------------------------------------------------- | 124 | // --------------------------------------------------------------------------- |
diff --git a/server/models/video/video-playlist.ts b/server/models/video/video-playlist.ts index 7dbe4ce8d..08e4d32c8 100644 --- a/server/models/video/video-playlist.ts +++ b/server/models/video/video-playlist.ts | |||
@@ -17,7 +17,7 @@ import { | |||
17 | } from 'sequelize-typescript' | 17 | } from 'sequelize-typescript' |
18 | import * as Sequelize from 'sequelize' | 18 | import * as Sequelize from 'sequelize' |
19 | import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model' | 19 | import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model' |
20 | import { buildServerIdsFollowedBy, buildWhereIdOrUUID, getSort, throwIfNotValid } from '../utils' | 20 | import { buildServerIdsFollowedBy, buildWhereIdOrUUID, getSort, isOutdated, throwIfNotValid } from '../utils' |
21 | import { | 21 | import { |
22 | isVideoPlaylistDescriptionValid, | 22 | isVideoPlaylistDescriptionValid, |
23 | isVideoPlaylistNameValid, | 23 | isVideoPlaylistNameValid, |
@@ -25,6 +25,7 @@ import { | |||
25 | } from '../../helpers/custom-validators/video-playlists' | 25 | } from '../../helpers/custom-validators/video-playlists' |
26 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' | 26 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' |
27 | import { | 27 | import { |
28 | ACTIVITY_PUB, | ||
28 | CONFIG, | 29 | CONFIG, |
29 | CONSTRAINTS_FIELDS, | 30 | CONSTRAINTS_FIELDS, |
30 | STATIC_PATHS, | 31 | STATIC_PATHS, |
@@ -429,10 +430,22 @@ export class VideoPlaylistModel extends Model<VideoPlaylistModel> { | |||
429 | .catch(err => logger.warn('Cannot delete thumbnail %s.', thumbnailPath, { err })) | 430 | .catch(err => logger.warn('Cannot delete thumbnail %s.', thumbnailPath, { err })) |
430 | } | 431 | } |
431 | 432 | ||
433 | setAsRefreshed () { | ||
434 | this.changed('updatedAt', true) | ||
435 | |||
436 | return this.save() | ||
437 | } | ||
438 | |||
432 | isOwned () { | 439 | isOwned () { |
433 | return this.OwnerAccount.isOwned() | 440 | return this.OwnerAccount.isOwned() |
434 | } | 441 | } |
435 | 442 | ||
443 | isOutdated () { | ||
444 | if (this.isOwned()) return false | ||
445 | |||
446 | return isOutdated(this, ACTIVITY_PUB.VIDEO_PLAYLIST_REFRESH_INTERVAL) | ||
447 | } | ||
448 | |||
436 | toFormattedJSON (): VideoPlaylist { | 449 | toFormattedJSON (): VideoPlaylist { |
437 | return { | 450 | return { |
438 | id: this.id, | 451 | id: this.id, |
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 946be6095..fb037c21a 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -77,7 +77,7 @@ import { | |||
77 | buildTrigramSearchIndex, | 77 | buildTrigramSearchIndex, |
78 | buildWhereIdOrUUID, | 78 | buildWhereIdOrUUID, |
79 | createSimilarityAttribute, | 79 | createSimilarityAttribute, |
80 | getVideoSort, | 80 | getVideoSort, isOutdated, |
81 | throwIfNotValid | 81 | throwIfNotValid |
82 | } from '../utils' | 82 | } from '../utils' |
83 | import { TagModel } from './tag' | 83 | import { TagModel } from './tag' |
@@ -1547,7 +1547,7 @@ export class VideoModel extends Model<VideoModel> { | |||
1547 | attributes: query.attributes, | 1547 | attributes: query.attributes, |
1548 | order: [ // Keep original order | 1548 | order: [ // Keep original order |
1549 | Sequelize.literal( | 1549 | Sequelize.literal( |
1550 | ids.map(id => `"VideoModel".id = ${id} DESC`).join(', ') | 1550 | ids.map(id => `"VideoModel".id = ${id}`).join(', ') |
1551 | ) | 1551 | ) |
1552 | ] | 1552 | ] |
1553 | } | 1553 | } |
@@ -1767,12 +1767,7 @@ export class VideoModel extends Model<VideoModel> { | |||
1767 | isOutdated () { | 1767 | isOutdated () { |
1768 | if (this.isOwned()) return false | 1768 | if (this.isOwned()) return false |
1769 | 1769 | ||
1770 | const now = Date.now() | 1770 | return isOutdated(this, ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL) |
1771 | const createdAtTime = this.createdAt.getTime() | ||
1772 | const updatedAtTime = this.updatedAt.getTime() | ||
1773 | |||
1774 | return (now - createdAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL && | ||
1775 | (now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL | ||
1776 | } | 1771 | } |
1777 | 1772 | ||
1778 | setAsRefreshed () { | 1773 | setAsRefreshed () { |
diff --git a/server/tests/api/activitypub/refresher.ts b/server/tests/api/activitypub/refresher.ts index 62ad8a0b5..ae4859076 100644 --- a/server/tests/api/activitypub/refresher.ts +++ b/server/tests/api/activitypub/refresher.ts | |||
@@ -2,89 +2,155 @@ | |||
2 | 2 | ||
3 | import 'mocha' | 3 | import 'mocha' |
4 | import { | 4 | import { |
5 | createVideoPlaylist, | ||
5 | doubleFollow, | 6 | doubleFollow, |
6 | flushAndRunMultipleServers, | 7 | flushAndRunMultipleServers, |
8 | generateUserAccessToken, | ||
7 | getVideo, | 9 | getVideo, |
10 | getVideoPlaylist, | ||
8 | killallServers, | 11 | killallServers, |
9 | reRunServer, | 12 | reRunServer, |
10 | ServerInfo, | 13 | ServerInfo, |
11 | setAccessTokensToServers, | 14 | setAccessTokensToServers, |
15 | setActorField, | ||
16 | setDefaultVideoChannel, | ||
17 | setPlaylistField, | ||
18 | setVideoField, | ||
12 | uploadVideo, | 19 | uploadVideo, |
20 | uploadVideoAndGetId, | ||
13 | wait, | 21 | wait, |
14 | setVideoField, | ||
15 | waitJobs | 22 | waitJobs |
16 | } from '../../../../shared/utils' | 23 | } from '../../../../shared/utils' |
24 | import { getAccount } from '../../../../shared/utils/users/accounts' | ||
25 | import { VideoPlaylistPrivacy } from '../../../../shared/models/videos' | ||
17 | 26 | ||
18 | describe('Test AP refresher', function () { | 27 | describe('Test AP refresher', function () { |
19 | let servers: ServerInfo[] = [] | 28 | let servers: ServerInfo[] = [] |
20 | let videoUUID1: string | 29 | let videoUUID1: string |
21 | let videoUUID2: string | 30 | let videoUUID2: string |
22 | let videoUUID3: string | 31 | let videoUUID3: string |
32 | let playlistUUID1: string | ||
33 | let playlistUUID2: string | ||
23 | 34 | ||
24 | before(async function () { | 35 | before(async function () { |
25 | this.timeout(60000) | 36 | this.timeout(60000) |
26 | 37 | ||
27 | servers = await flushAndRunMultipleServers(2) | 38 | servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: false } }) |
28 | 39 | ||
29 | // Get the access tokens | 40 | // Get the access tokens |
30 | await setAccessTokensToServers(servers) | 41 | await setAccessTokensToServers(servers) |
42 | await setDefaultVideoChannel(servers) | ||
43 | |||
44 | { | ||
45 | videoUUID1 = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video1' })).uuid | ||
46 | videoUUID2 = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video2' })).uuid | ||
47 | videoUUID3 = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video3' })).uuid | ||
48 | } | ||
31 | 49 | ||
32 | { | 50 | { |
33 | const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video1' }) | 51 | const a1 = await generateUserAccessToken(servers[1], 'user1') |
34 | videoUUID1 = res.body.video.uuid | 52 | await uploadVideo(servers[1].url, a1, { name: 'video4' }) |
53 | |||
54 | const a2 = await generateUserAccessToken(servers[1], 'user2') | ||
55 | await uploadVideo(servers[1].url, a2, { name: 'video5' }) | ||
35 | } | 56 | } |
36 | 57 | ||
37 | { | 58 | { |
38 | const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video2' }) | 59 | const playlistAttrs = { displayName: 'playlist1', privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: servers[1].videoChannel.id } |
39 | videoUUID2 = res.body.video.uuid | 60 | const res = await createVideoPlaylist({ url: servers[1].url, token: servers[1].accessToken, playlistAttrs }) |
61 | playlistUUID1 = res.body.videoPlaylist.uuid | ||
40 | } | 62 | } |
41 | 63 | ||
42 | { | 64 | { |
43 | const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video3' }) | 65 | const playlistAttrs = { displayName: 'playlist2', privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: servers[1].videoChannel.id } |
44 | videoUUID3 = res.body.video.uuid | 66 | const res = await createVideoPlaylist({ url: servers[1].url, token: servers[1].accessToken, playlistAttrs }) |
67 | playlistUUID2 = res.body.videoPlaylist.uuid | ||
45 | } | 68 | } |
46 | 69 | ||
47 | await doubleFollow(servers[0], servers[1]) | 70 | await doubleFollow(servers[0], servers[1]) |
48 | }) | 71 | }) |
49 | 72 | ||
50 | it('Should remove a deleted remote video', async function () { | 73 | describe('Videos refresher', function () { |
51 | this.timeout(60000) | 74 | |
75 | it('Should remove a deleted remote video', async function () { | ||
76 | this.timeout(60000) | ||
77 | |||
78 | await wait(10000) | ||
79 | |||
80 | // Change UUID so the remote server returns a 404 | ||
81 | await setVideoField(2, videoUUID1, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174f') | ||
82 | |||
83 | await getVideo(servers[ 0 ].url, videoUUID1) | ||
84 | await getVideo(servers[ 0 ].url, videoUUID2) | ||
85 | |||
86 | await waitJobs(servers) | ||
52 | 87 | ||
53 | await wait(10000) | 88 | await getVideo(servers[ 0 ].url, videoUUID1, 404) |
89 | await getVideo(servers[ 0 ].url, videoUUID2, 200) | ||
90 | }) | ||
54 | 91 | ||
55 | // Change UUID so the remote server returns a 404 | 92 | it('Should not update a remote video if the remote instance is down', async function () { |
56 | await setVideoField(2, videoUUID1, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174f') | 93 | this.timeout(60000) |
57 | 94 | ||
58 | await getVideo(servers[0].url, videoUUID1) | 95 | killallServers([ servers[ 1 ] ]) |
59 | await getVideo(servers[0].url, videoUUID2) | ||
60 | 96 | ||
61 | await waitJobs(servers) | 97 | await setVideoField(2, videoUUID3, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174e') |
62 | 98 | ||
63 | await getVideo(servers[0].url, videoUUID1, 404) | 99 | // Video will need a refresh |
64 | await getVideo(servers[0].url, videoUUID2, 200) | 100 | await wait(10000) |
101 | |||
102 | await getVideo(servers[ 0 ].url, videoUUID3) | ||
103 | // The refresh should fail | ||
104 | await waitJobs([ servers[ 0 ] ]) | ||
105 | |||
106 | await reRunServer(servers[ 1 ]) | ||
107 | |||
108 | // Should not refresh the video, even if the last refresh failed (to avoir a loop on dead instances) | ||
109 | await getVideo(servers[ 0 ].url, videoUUID3) | ||
110 | await waitJobs(servers) | ||
111 | |||
112 | await getVideo(servers[ 0 ].url, videoUUID3, 200) | ||
113 | }) | ||
65 | }) | 114 | }) |
66 | 115 | ||
67 | it('Should not update a remote video if the remote instance is down', async function () { | 116 | describe('Actors refresher', function () { |
68 | this.timeout(60000) | 117 | |
118 | it('Should remove a deleted actor', async function () { | ||
119 | this.timeout(60000) | ||
120 | |||
121 | await wait(10000) | ||
122 | |||
123 | // Change actor name so the remote server returns a 404 | ||
124 | await setActorField(2, 'http://localhost:9002/accounts/user2', 'preferredUsername', 'toto') | ||
125 | |||
126 | await getAccount(servers[ 0 ].url, 'user1@localhost:9002') | ||
127 | await getAccount(servers[ 0 ].url, 'user2@localhost:9002') | ||
128 | |||
129 | await waitJobs(servers) | ||
130 | |||
131 | await getAccount(servers[ 0 ].url, 'user1@localhost:9002', 200) | ||
132 | await getAccount(servers[ 0 ].url, 'user2@localhost:9002', 404) | ||
133 | }) | ||
134 | }) | ||
69 | 135 | ||
70 | killallServers([ servers[1] ]) | 136 | describe('Playlist refresher', function () { |
71 | 137 | ||
72 | await setVideoField(2, videoUUID3, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174e') | 138 | it('Should remove a deleted playlist', async function () { |
139 | this.timeout(60000) | ||
73 | 140 | ||
74 | // Video will need a refresh | 141 | await wait(10000) |
75 | await wait(10000) | ||
76 | 142 | ||
77 | await getVideo(servers[0].url, videoUUID3) | 143 | // Change UUID so the remote server returns a 404 |
78 | // The refresh should fail | 144 | await setPlaylistField(2, playlistUUID2, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b178e') |
79 | await waitJobs([ servers[0] ]) | ||
80 | 145 | ||
81 | await reRunServer(servers[1]) | 146 | await getVideoPlaylist(servers[ 0 ].url, playlistUUID1) |
147 | await getVideoPlaylist(servers[ 0 ].url, playlistUUID2) | ||
82 | 148 | ||
83 | // Should not refresh the video, even if the last refresh failed (to avoir a loop on dead instances) | 149 | await waitJobs(servers) |
84 | await getVideo(servers[0].url, videoUUID3) | ||
85 | await waitJobs(servers) | ||
86 | 150 | ||
87 | await getVideo(servers[0].url, videoUUID3, 200) | 151 | await getVideoPlaylist(servers[ 0 ].url, playlistUUID1, 200) |
152 | await getVideoPlaylist(servers[ 0 ].url, playlistUUID2, 404) | ||
153 | }) | ||
88 | }) | 154 | }) |
89 | 155 | ||
90 | after(async function () { | 156 | after(async function () { |
diff --git a/shared/utils/miscs/sql.ts b/shared/utils/miscs/sql.ts index bb3f63837..1ce3d801a 100644 --- a/shared/utils/miscs/sql.ts +++ b/shared/utils/miscs/sql.ts | |||
@@ -40,6 +40,14 @@ function setVideoField (serverNumber: number, uuid: string, field: string, value | |||
40 | return seq.query(`UPDATE video SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options) | 40 | return seq.query(`UPDATE video SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options) |
41 | } | 41 | } |
42 | 42 | ||
43 | function setPlaylistField (serverNumber: number, uuid: string, field: string, value: string) { | ||
44 | const seq = getSequelize(serverNumber) | ||
45 | |||
46 | const options = { type: Sequelize.QueryTypes.UPDATE } | ||
47 | |||
48 | return seq.query(`UPDATE "videoPlaylist" SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options) | ||
49 | } | ||
50 | |||
43 | async function closeAllSequelize (servers: any[]) { | 51 | async function closeAllSequelize (servers: any[]) { |
44 | for (let i = 1; i <= servers.length; i++) { | 52 | for (let i = 1; i <= servers.length; i++) { |
45 | if (sequelizes[ i ]) { | 53 | if (sequelizes[ i ]) { |
@@ -51,6 +59,7 @@ async function closeAllSequelize (servers: any[]) { | |||
51 | 59 | ||
52 | export { | 60 | export { |
53 | setVideoField, | 61 | setVideoField, |
62 | setPlaylistField, | ||
54 | setActorField, | 63 | setActorField, |
55 | closeAllSequelize | 64 | closeAllSequelize |
56 | } | 65 | } |
diff --git a/shared/utils/videos/video-playlists.ts b/shared/utils/videos/video-playlists.ts index 7568852dc..4d110a131 100644 --- a/shared/utils/videos/video-playlists.ts +++ b/shared/utils/videos/video-playlists.ts | |||
@@ -127,7 +127,7 @@ function createVideoPlaylist (options: { | |||
127 | playlistAttrs: VideoPlaylistCreate, | 127 | playlistAttrs: VideoPlaylistCreate, |
128 | expectedStatus?: number | 128 | expectedStatus?: number |
129 | }) { | 129 | }) { |
130 | const path = '/api/v1/video-playlists/' | 130 | const path = '/api/v1/video-playlists' |
131 | 131 | ||
132 | const fields = omit(options.playlistAttrs, 'thumbnailfile') | 132 | const fields = omit(options.playlistAttrs, 'thumbnailfile') |
133 | 133 | ||