aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2019-03-19 14:13:53 +0100
committerChocobozzz <me@florianbigard.com>2019-03-19 14:13:53 +0100
commit9f79ade627f0044606a9fbbe16ca0154661d12b9 (patch)
treeccfc1876da591bc52cbb4b8ccddedbeae7876679
parent0e0c745b62b2f18e228328e8c4f95b17e54f7a5e (diff)
downloadPeerTube-9f79ade627f0044606a9fbbe16ca0154661d12b9.tar.gz
PeerTube-9f79ade627f0044606a9fbbe16ca0154661d12b9.tar.zst
PeerTube-9f79ade627f0044606a9fbbe16ca0154661d12b9.zip
Refresh playlists
-rw-r--r--server/controllers/api/video-playlist.ts6
-rw-r--r--server/initializers/constants.ts4
-rw-r--r--server/lib/activitypub/actor.ts3
-rw-r--r--server/lib/activitypub/playlist.ts56
-rw-r--r--server/lib/job-queue/handlers/activitypub-refresher.ts14
-rw-r--r--server/middlewares/validators/videos/video-playlists.ts3
-rw-r--r--server/models/activitypub/actor.ts9
-rw-r--r--server/models/utils.ts12
-rw-r--r--server/models/video/video-playlist.ts15
-rw-r--r--server/models/video/video.ts11
-rw-r--r--server/tests/api/activitypub/refresher.ts130
-rw-r--r--shared/utils/miscs/sql.ts9
-rw-r--r--shared/utils/videos/video-playlists.ts2
13 files changed, 215 insertions, 59 deletions
diff --git a/server/controllers/api/video-playlist.ts b/server/controllers/api/video-playlist.ts
index 5b1601c4e..feba30564 100644
--- a/server/controllers/api/video-playlist.ts
+++ b/server/controllers/api/video-playlist.ts
@@ -40,6 +40,7 @@ import { VideoPlaylistElementUpdate } from '../../../shared/models/videos/playli
40import { copy, pathExists } from 'fs-extra' 40import { copy, pathExists } from 'fs-extra'
41import { AccountModel } from '../../models/account/account' 41import { AccountModel } from '../../models/account/account'
42import { VideoPlaylistReorder } from '../../../shared/models/videos/playlist/video-playlist-reorder.model' 42import { VideoPlaylistReorder } from '../../../shared/models/videos/playlist/video-playlist-reorder.model'
43import { JobQueue } from '../../lib/job-queue'
43 44
44const reqThumbnailFile = createReqFiles([ 'thumbnailfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, { thumbnailfile: CONFIG.STORAGE.TMP_DIR }) 45const reqThumbnailFile = createReqFiles([ 'thumbnailfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, { thumbnailfile: CONFIG.STORAGE.TMP_DIR })
45 46
@@ -142,6 +143,11 @@ async function listVideoPlaylists (req: express.Request, res: express.Response)
142function getVideoPlaylist (req: express.Request, res: express.Response) { 143function getVideoPlaylist (req: express.Request, res: express.Response) {
143 const videoPlaylist = res.locals.videoPlaylist 144 const videoPlaylist = res.locals.videoPlaylist
144 145
146 if (videoPlaylist.isOutdated()) {
147 JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video-playlist', url: videoPlaylist.url } })
148 .catch(err => logger.error('Cannot create AP refresher job for playlist %s.', videoPlaylist.url, { err }))
149 }
150
145 return res.json(videoPlaylist.toFormattedJSON()) 151 return res.json(videoPlaylist.toFormattedJSON())
146} 152}
147 153
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 19e63d6fe..7fac8a4d6 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -584,7 +584,8 @@ const ACTIVITY_PUB = {
584 }, 584 },
585 MAX_RECURSION_COMMENTS: 100, 585 MAX_RECURSION_COMMENTS: 100,
586 ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days 586 ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days
587 VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2 // 2 days 587 VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days
588 VIDEO_PLAYLIST_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2 // 2 days
588} 589}
589 590
590const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = { 591const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = {
@@ -724,6 +725,7 @@ if (isTestInstance() === true) {
724 ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2 725 ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2
725 ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 seconds 726 ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
726 ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL = 10 * 1000 // 10 seconds 727 ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
728 ACTIVITY_PUB.VIDEO_PLAYLIST_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
727 729
728 CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB 730 CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB
729 731
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index f77df8b78..63e810642 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -375,7 +375,8 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe
375 } 375 }
376 376
377 if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) { 377 if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
378 throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id) 378 logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id)
379 return { result: undefined, statusCode: requestResult.response.statusCode }
379 } 380 }
380 381
381 const followersCount = await fetchActorTotalItems(actorJSON.followers) 382 const followersCount = await fetchActorTotalItems(actorJSON.followers)
diff --git a/server/lib/activitypub/playlist.ts b/server/lib/activitypub/playlist.ts
index 70389044e..c4a8f12ec 100644
--- a/server/lib/activitypub/playlist.ts
+++ b/server/lib/activitypub/playlist.ts
@@ -95,7 +95,7 @@ async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAc
95 return Promise.resolve() 95 return Promise.resolve()
96 }) 96 })
97 97
98 // Empty playlists generally do not have a miniature, so skip it 98 // Empty playlists generally do not have a miniature, so skip this
99 if (accItems.length !== 0) { 99 if (accItems.length !== 0) {
100 try { 100 try {
101 await generateThumbnailFromUrl(playlist, playlistObject.icon) 101 await generateThumbnailFromUrl(playlist, playlistObject.icon)
@@ -107,13 +107,45 @@ async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAc
107 return resetVideoPlaylistElements(accItems, playlist) 107 return resetVideoPlaylistElements(accItems, playlist)
108} 108}
109 109
110async function refreshVideoPlaylistIfNeeded (videoPlaylist: VideoPlaylistModel): Promise<VideoPlaylistModel> {
111 if (!videoPlaylist.isOutdated()) return videoPlaylist
112
113 try {
114 const { statusCode, playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
115 if (statusCode === 404) {
116 logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
117
118 await videoPlaylist.destroy()
119 return undefined
120 }
121
122 if (playlistObject === undefined) {
123 logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url)
124
125 await videoPlaylist.setAsRefreshed()
126 return videoPlaylist
127 }
128
129 const byAccount = videoPlaylist.OwnerAccount
130 await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
131
132 return videoPlaylist
133 } catch (err) {
134 logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err })
135
136 await videoPlaylist.setAsRefreshed()
137 return videoPlaylist
138 }
139}
140
110// --------------------------------------------------------------------------- 141// ---------------------------------------------------------------------------
111 142
112export { 143export {
113 createAccountPlaylists, 144 createAccountPlaylists,
114 playlistObjectToDBAttributes, 145 playlistObjectToDBAttributes,
115 playlistElementObjectToDBAttributes, 146 playlistElementObjectToDBAttributes,
116 createOrUpdateVideoPlaylist 147 createOrUpdateVideoPlaylist,
148 refreshVideoPlaylistIfNeeded
117} 149}
118 150
119// --------------------------------------------------------------------------- 151// ---------------------------------------------------------------------------
@@ -162,3 +194,23 @@ function generateThumbnailFromUrl (playlist: VideoPlaylistModel, icon: ActivityI
162 194
163 return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE) 195 return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE)
164} 196}
197
198async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
199 const options = {
200 uri: playlistUrl,
201 method: 'GET',
202 json: true,
203 activityPub: true
204 }
205
206 logger.info('Fetching remote playlist %s.', playlistUrl)
207
208 const { response, body } = await doRequest(options)
209
210 if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
211 logger.debug('Remote video playlist JSON is not valid.', { body })
212 return { statusCode: response.statusCode, playlistObject: undefined }
213 }
214
215 return { statusCode: response.statusCode, playlistObject: body }
216}
diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts
index 454b975fe..4d6c38cfa 100644
--- a/server/lib/job-queue/handlers/activitypub-refresher.ts
+++ b/server/lib/job-queue/handlers/activitypub-refresher.ts
@@ -1,11 +1,12 @@
1import * as Bull from 'bull' 1import * as Bull from 'bull'
2import { logger } from '../../../helpers/logger' 2import { logger } from '../../../helpers/logger'
3import { fetchVideoByUrl } from '../../../helpers/video' 3import { fetchVideoByUrl } from '../../../helpers/video'
4import { refreshVideoIfNeeded, refreshActorIfNeeded } from '../../activitypub' 4import { refreshActorIfNeeded, refreshVideoIfNeeded, refreshVideoPlaylistIfNeeded } from '../../activitypub'
5import { ActorModel } from '../../../models/activitypub/actor' 5import { ActorModel } from '../../../models/activitypub/actor'
6import { VideoPlaylistModel } from '../../../models/video/video-playlist'
6 7
7export type RefreshPayload = { 8export type RefreshPayload = {
8 type: 'video' | 'actor' 9 type: 'video' | 'video-playlist' | 'actor'
9 url: string 10 url: string
10} 11}
11 12
@@ -15,13 +16,13 @@ async function refreshAPObject (job: Bull.Job) {
15 logger.info('Processing AP refresher in job %d for %s.', job.id, payload.url) 16 logger.info('Processing AP refresher in job %d for %s.', job.id, payload.url)
16 17
17 if (payload.type === 'video') return refreshVideo(payload.url) 18 if (payload.type === 'video') return refreshVideo(payload.url)
19 if (payload.type === 'video-playlist') return refreshVideoPlaylist(payload.url)
18 if (payload.type === 'actor') return refreshActor(payload.url) 20 if (payload.type === 'actor') return refreshActor(payload.url)
19} 21}
20 22
21// --------------------------------------------------------------------------- 23// ---------------------------------------------------------------------------
22 24
23export { 25export {
24 refreshActor,
25 refreshAPObject 26 refreshAPObject
26} 27}
27 28
@@ -50,5 +51,12 @@ async function refreshActor (actorUrl: string) {
50 if (actor) { 51 if (actor) {
51 await refreshActorIfNeeded(actor, fetchType) 52 await refreshActorIfNeeded(actor, fetchType)
52 } 53 }
54}
55
56async function refreshVideoPlaylist (playlistUrl: string) {
57 const playlist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(playlistUrl)
53 58
59 if (playlist) {
60 await refreshVideoPlaylistIfNeeded(playlist)
61 }
54} 62}
diff --git a/server/middlewares/validators/videos/video-playlists.ts b/server/middlewares/validators/videos/video-playlists.ts
index 55e09e354..6ba30fac9 100644
--- a/server/middlewares/validators/videos/video-playlists.ts
+++ b/server/middlewares/validators/videos/video-playlists.ts
@@ -8,8 +8,8 @@ import { doesVideoExist, isVideoImage } from '../../../helpers/custom-validators
8import { CONSTRAINTS_FIELDS } from '../../../initializers' 8import { CONSTRAINTS_FIELDS } from '../../../initializers'
9import { isArrayOf, isIdOrUUIDValid, isIdValid, isUUIDValid, toIntArray, toValueOrNull } from '../../../helpers/custom-validators/misc' 9import { isArrayOf, isIdOrUUIDValid, isIdValid, isUUIDValid, toIntArray, toValueOrNull } from '../../../helpers/custom-validators/misc'
10import { 10import {
11 isVideoPlaylistDescriptionValid,
12 doesVideoPlaylistExist, 11 doesVideoPlaylistExist,
12 isVideoPlaylistDescriptionValid,
13 isVideoPlaylistNameValid, 13 isVideoPlaylistNameValid,
14 isVideoPlaylistPrivacyValid, 14 isVideoPlaylistPrivacyValid,
15 isVideoPlaylistTimestampValid, 15 isVideoPlaylistTimestampValid,
@@ -19,7 +19,6 @@ import { VideoPlaylistModel } from '../../../models/video/video-playlist'
19import { cleanUpReqFiles } from '../../../helpers/express-utils' 19import { cleanUpReqFiles } from '../../../helpers/express-utils'
20import { doesVideoChannelIdExist } from '../../../helpers/custom-validators/video-channels' 20import { doesVideoChannelIdExist } from '../../../helpers/custom-validators/video-channels'
21import { VideoPlaylistElementModel } from '../../../models/video/video-playlist-element' 21import { VideoPlaylistElementModel } from '../../../models/video/video-playlist-element'
22import { VideoModel } from '../../../models/video/video'
23import { authenticatePromiseIfNeeded } from '../../oauth' 22import { authenticatePromiseIfNeeded } from '../../oauth'
24import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model' 23import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model'
25import { VideoPlaylistType } from '../../../../shared/models/videos/playlist/video-playlist-type.model' 24import { VideoPlaylistType } from '../../../../shared/models/videos/playlist/video-playlist-type.model'
diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts
index 2fceb21dd..7d91e8a4a 100644
--- a/server/models/activitypub/actor.ts
+++ b/server/models/activitypub/actor.ts
@@ -34,7 +34,7 @@ import { ACTIVITY_PUB, ACTIVITY_PUB_ACTOR_TYPES, CONFIG, CONSTRAINTS_FIELDS } fr
34import { AccountModel } from '../account/account' 34import { AccountModel } from '../account/account'
35import { AvatarModel } from '../avatar/avatar' 35import { AvatarModel } from '../avatar/avatar'
36import { ServerModel } from '../server/server' 36import { ServerModel } from '../server/server'
37import { throwIfNotValid } from '../utils' 37import { isOutdated, throwIfNotValid } from '../utils'
38import { VideoChannelModel } from '../video/video-channel' 38import { VideoChannelModel } from '../video/video-channel'
39import { ActorFollowModel } from './actor-follow' 39import { ActorFollowModel } from './actor-follow'
40import { VideoModel } from '../video/video' 40import { VideoModel } from '../video/video'
@@ -532,11 +532,6 @@ export class ActorModel extends Model<ActorModel> {
532 isOutdated () { 532 isOutdated () {
533 if (this.isOwned()) return false 533 if (this.isOwned()) return false
534 534
535 const now = Date.now() 535 return isOutdated(this, ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL)
536 const createdAtTime = this.createdAt.getTime()
537 const updatedAtTime = this.updatedAt.getTime()
538
539 return (now - createdAtTime) > ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL &&
540 (now - updatedAtTime) > ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL
541 } 536 }
542} 537}
diff --git a/server/models/utils.ts b/server/models/utils.ts
index 4ebd07dab..f8a71b270 100644
--- a/server/models/utils.ts
+++ b/server/models/utils.ts
@@ -1,5 +1,6 @@
1import { Sequelize } from 'sequelize-typescript' 1import { Sequelize } from 'sequelize-typescript'
2import * as validator from 'validator' 2import * as validator from 'validator'
3import { ACTIVITY_PUB } from '../initializers'
3 4
4type SortType = { sortModel: any, sortValue: string } 5type SortType = { sortModel: any, sortValue: string }
5 6
@@ -44,6 +45,14 @@ function getSortOnModel (model: any, value: string, lastSort: string[] = [ 'id',
44 return [ firstSort, lastSort ] 45 return [ firstSort, lastSort ]
45} 46}
46 47
48function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) {
49 const now = Date.now()
50 const createdAtTime = model.createdAt.getTime()
51 const updatedAtTime = model.updatedAt.getTime()
52
53 return (now - createdAtTime) > refreshInterval && (now - updatedAtTime) > refreshInterval
54}
55
47function throwIfNotValid (value: any, validator: (value: any) => boolean, fieldName = 'value') { 56function throwIfNotValid (value: any, validator: (value: any) => boolean, fieldName = 'value') {
48 if (validator(value) === false) { 57 if (validator(value) === false) {
49 throw new Error(`"${value}" is not a valid ${fieldName}.`) 58 throw new Error(`"${value}" is not a valid ${fieldName}.`)
@@ -108,7 +117,8 @@ export {
108 throwIfNotValid, 117 throwIfNotValid,
109 buildServerIdsFollowedBy, 118 buildServerIdsFollowedBy,
110 buildTrigramSearchIndex, 119 buildTrigramSearchIndex,
111 buildWhereIdOrUUID 120 buildWhereIdOrUUID,
121 isOutdated
112} 122}
113 123
114// --------------------------------------------------------------------------- 124// ---------------------------------------------------------------------------
diff --git a/server/models/video/video-playlist.ts b/server/models/video/video-playlist.ts
index 7dbe4ce8d..08e4d32c8 100644
--- a/server/models/video/video-playlist.ts
+++ b/server/models/video/video-playlist.ts
@@ -17,7 +17,7 @@ import {
17} from 'sequelize-typescript' 17} from 'sequelize-typescript'
18import * as Sequelize from 'sequelize' 18import * as Sequelize from 'sequelize'
19import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model' 19import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
20import { buildServerIdsFollowedBy, buildWhereIdOrUUID, getSort, throwIfNotValid } from '../utils' 20import { buildServerIdsFollowedBy, buildWhereIdOrUUID, getSort, isOutdated, throwIfNotValid } from '../utils'
21import { 21import {
22 isVideoPlaylistDescriptionValid, 22 isVideoPlaylistDescriptionValid,
23 isVideoPlaylistNameValid, 23 isVideoPlaylistNameValid,
@@ -25,6 +25,7 @@ import {
25} from '../../helpers/custom-validators/video-playlists' 25} from '../../helpers/custom-validators/video-playlists'
26import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' 26import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
27import { 27import {
28 ACTIVITY_PUB,
28 CONFIG, 29 CONFIG,
29 CONSTRAINTS_FIELDS, 30 CONSTRAINTS_FIELDS,
30 STATIC_PATHS, 31 STATIC_PATHS,
@@ -429,10 +430,22 @@ export class VideoPlaylistModel extends Model<VideoPlaylistModel> {
429 .catch(err => logger.warn('Cannot delete thumbnail %s.', thumbnailPath, { err })) 430 .catch(err => logger.warn('Cannot delete thumbnail %s.', thumbnailPath, { err }))
430 } 431 }
431 432
433 setAsRefreshed () {
434 this.changed('updatedAt', true)
435
436 return this.save()
437 }
438
432 isOwned () { 439 isOwned () {
433 return this.OwnerAccount.isOwned() 440 return this.OwnerAccount.isOwned()
434 } 441 }
435 442
443 isOutdated () {
444 if (this.isOwned()) return false
445
446 return isOutdated(this, ACTIVITY_PUB.VIDEO_PLAYLIST_REFRESH_INTERVAL)
447 }
448
436 toFormattedJSON (): VideoPlaylist { 449 toFormattedJSON (): VideoPlaylist {
437 return { 450 return {
438 id: this.id, 451 id: this.id,
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 946be6095..fb037c21a 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -77,7 +77,7 @@ import {
77 buildTrigramSearchIndex, 77 buildTrigramSearchIndex,
78 buildWhereIdOrUUID, 78 buildWhereIdOrUUID,
79 createSimilarityAttribute, 79 createSimilarityAttribute,
80 getVideoSort, 80 getVideoSort, isOutdated,
81 throwIfNotValid 81 throwIfNotValid
82} from '../utils' 82} from '../utils'
83import { TagModel } from './tag' 83import { TagModel } from './tag'
@@ -1547,7 +1547,7 @@ export class VideoModel extends Model<VideoModel> {
1547 attributes: query.attributes, 1547 attributes: query.attributes,
1548 order: [ // Keep original order 1548 order: [ // Keep original order
1549 Sequelize.literal( 1549 Sequelize.literal(
1550 ids.map(id => `"VideoModel".id = ${id} DESC`).join(', ') 1550 ids.map(id => `"VideoModel".id = ${id}`).join(', ')
1551 ) 1551 )
1552 ] 1552 ]
1553 } 1553 }
@@ -1767,12 +1767,7 @@ export class VideoModel extends Model<VideoModel> {
1767 isOutdated () { 1767 isOutdated () {
1768 if (this.isOwned()) return false 1768 if (this.isOwned()) return false
1769 1769
1770 const now = Date.now() 1770 return isOutdated(this, ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL)
1771 const createdAtTime = this.createdAt.getTime()
1772 const updatedAtTime = this.updatedAt.getTime()
1773
1774 return (now - createdAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL &&
1775 (now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL
1776 } 1771 }
1777 1772
1778 setAsRefreshed () { 1773 setAsRefreshed () {
diff --git a/server/tests/api/activitypub/refresher.ts b/server/tests/api/activitypub/refresher.ts
index 62ad8a0b5..ae4859076 100644
--- a/server/tests/api/activitypub/refresher.ts
+++ b/server/tests/api/activitypub/refresher.ts
@@ -2,89 +2,155 @@
2 2
3import 'mocha' 3import 'mocha'
4import { 4import {
5 createVideoPlaylist,
5 doubleFollow, 6 doubleFollow,
6 flushAndRunMultipleServers, 7 flushAndRunMultipleServers,
8 generateUserAccessToken,
7 getVideo, 9 getVideo,
10 getVideoPlaylist,
8 killallServers, 11 killallServers,
9 reRunServer, 12 reRunServer,
10 ServerInfo, 13 ServerInfo,
11 setAccessTokensToServers, 14 setAccessTokensToServers,
15 setActorField,
16 setDefaultVideoChannel,
17 setPlaylistField,
18 setVideoField,
12 uploadVideo, 19 uploadVideo,
20 uploadVideoAndGetId,
13 wait, 21 wait,
14 setVideoField,
15 waitJobs 22 waitJobs
16} from '../../../../shared/utils' 23} from '../../../../shared/utils'
24import { getAccount } from '../../../../shared/utils/users/accounts'
25import { VideoPlaylistPrivacy } from '../../../../shared/models/videos'
17 26
18describe('Test AP refresher', function () { 27describe('Test AP refresher', function () {
19 let servers: ServerInfo[] = [] 28 let servers: ServerInfo[] = []
20 let videoUUID1: string 29 let videoUUID1: string
21 let videoUUID2: string 30 let videoUUID2: string
22 let videoUUID3: string 31 let videoUUID3: string
32 let playlistUUID1: string
33 let playlistUUID2: string
23 34
24 before(async function () { 35 before(async function () {
25 this.timeout(60000) 36 this.timeout(60000)
26 37
27 servers = await flushAndRunMultipleServers(2) 38 servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: false } })
28 39
29 // Get the access tokens 40 // Get the access tokens
30 await setAccessTokensToServers(servers) 41 await setAccessTokensToServers(servers)
42 await setDefaultVideoChannel(servers)
43
44 {
45 videoUUID1 = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video1' })).uuid
46 videoUUID2 = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video2' })).uuid
47 videoUUID3 = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video3' })).uuid
48 }
31 49
32 { 50 {
33 const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video1' }) 51 const a1 = await generateUserAccessToken(servers[1], 'user1')
34 videoUUID1 = res.body.video.uuid 52 await uploadVideo(servers[1].url, a1, { name: 'video4' })
53
54 const a2 = await generateUserAccessToken(servers[1], 'user2')
55 await uploadVideo(servers[1].url, a2, { name: 'video5' })
35 } 56 }
36 57
37 { 58 {
38 const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video2' }) 59 const playlistAttrs = { displayName: 'playlist1', privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: servers[1].videoChannel.id }
39 videoUUID2 = res.body.video.uuid 60 const res = await createVideoPlaylist({ url: servers[1].url, token: servers[1].accessToken, playlistAttrs })
61 playlistUUID1 = res.body.videoPlaylist.uuid
40 } 62 }
41 63
42 { 64 {
43 const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video3' }) 65 const playlistAttrs = { displayName: 'playlist2', privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: servers[1].videoChannel.id }
44 videoUUID3 = res.body.video.uuid 66 const res = await createVideoPlaylist({ url: servers[1].url, token: servers[1].accessToken, playlistAttrs })
67 playlistUUID2 = res.body.videoPlaylist.uuid
45 } 68 }
46 69
47 await doubleFollow(servers[0], servers[1]) 70 await doubleFollow(servers[0], servers[1])
48 }) 71 })
49 72
50 it('Should remove a deleted remote video', async function () { 73 describe('Videos refresher', function () {
51 this.timeout(60000) 74
75 it('Should remove a deleted remote video', async function () {
76 this.timeout(60000)
77
78 await wait(10000)
79
80 // Change UUID so the remote server returns a 404
81 await setVideoField(2, videoUUID1, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174f')
82
83 await getVideo(servers[ 0 ].url, videoUUID1)
84 await getVideo(servers[ 0 ].url, videoUUID2)
85
86 await waitJobs(servers)
52 87
53 await wait(10000) 88 await getVideo(servers[ 0 ].url, videoUUID1, 404)
89 await getVideo(servers[ 0 ].url, videoUUID2, 200)
90 })
54 91
55 // Change UUID so the remote server returns a 404 92 it('Should not update a remote video if the remote instance is down', async function () {
56 await setVideoField(2, videoUUID1, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174f') 93 this.timeout(60000)
57 94
58 await getVideo(servers[0].url, videoUUID1) 95 killallServers([ servers[ 1 ] ])
59 await getVideo(servers[0].url, videoUUID2)
60 96
61 await waitJobs(servers) 97 await setVideoField(2, videoUUID3, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174e')
62 98
63 await getVideo(servers[0].url, videoUUID1, 404) 99 // Video will need a refresh
64 await getVideo(servers[0].url, videoUUID2, 200) 100 await wait(10000)
101
102 await getVideo(servers[ 0 ].url, videoUUID3)
103 // The refresh should fail
104 await waitJobs([ servers[ 0 ] ])
105
106 await reRunServer(servers[ 1 ])
107
108 // Should not refresh the video, even if the last refresh failed (to avoir a loop on dead instances)
109 await getVideo(servers[ 0 ].url, videoUUID3)
110 await waitJobs(servers)
111
112 await getVideo(servers[ 0 ].url, videoUUID3, 200)
113 })
65 }) 114 })
66 115
67 it('Should not update a remote video if the remote instance is down', async function () { 116 describe('Actors refresher', function () {
68 this.timeout(60000) 117
118 it('Should remove a deleted actor', async function () {
119 this.timeout(60000)
120
121 await wait(10000)
122
123 // Change actor name so the remote server returns a 404
124 await setActorField(2, 'http://localhost:9002/accounts/user2', 'preferredUsername', 'toto')
125
126 await getAccount(servers[ 0 ].url, 'user1@localhost:9002')
127 await getAccount(servers[ 0 ].url, 'user2@localhost:9002')
128
129 await waitJobs(servers)
130
131 await getAccount(servers[ 0 ].url, 'user1@localhost:9002', 200)
132 await getAccount(servers[ 0 ].url, 'user2@localhost:9002', 404)
133 })
134 })
69 135
70 killallServers([ servers[1] ]) 136 describe('Playlist refresher', function () {
71 137
72 await setVideoField(2, videoUUID3, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174e') 138 it('Should remove a deleted playlist', async function () {
139 this.timeout(60000)
73 140
74 // Video will need a refresh 141 await wait(10000)
75 await wait(10000)
76 142
77 await getVideo(servers[0].url, videoUUID3) 143 // Change UUID so the remote server returns a 404
78 // The refresh should fail 144 await setPlaylistField(2, playlistUUID2, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b178e')
79 await waitJobs([ servers[0] ])
80 145
81 await reRunServer(servers[1]) 146 await getVideoPlaylist(servers[ 0 ].url, playlistUUID1)
147 await getVideoPlaylist(servers[ 0 ].url, playlistUUID2)
82 148
83 // Should not refresh the video, even if the last refresh failed (to avoir a loop on dead instances) 149 await waitJobs(servers)
84 await getVideo(servers[0].url, videoUUID3)
85 await waitJobs(servers)
86 150
87 await getVideo(servers[0].url, videoUUID3, 200) 151 await getVideoPlaylist(servers[ 0 ].url, playlistUUID1, 200)
152 await getVideoPlaylist(servers[ 0 ].url, playlistUUID2, 404)
153 })
88 }) 154 })
89 155
90 after(async function () { 156 after(async function () {
diff --git a/shared/utils/miscs/sql.ts b/shared/utils/miscs/sql.ts
index bb3f63837..1ce3d801a 100644
--- a/shared/utils/miscs/sql.ts
+++ b/shared/utils/miscs/sql.ts
@@ -40,6 +40,14 @@ function setVideoField (serverNumber: number, uuid: string, field: string, value
40 return seq.query(`UPDATE video SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options) 40 return seq.query(`UPDATE video SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options)
41} 41}
42 42
43function setPlaylistField (serverNumber: number, uuid: string, field: string, value: string) {
44 const seq = getSequelize(serverNumber)
45
46 const options = { type: Sequelize.QueryTypes.UPDATE }
47
48 return seq.query(`UPDATE "videoPlaylist" SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options)
49}
50
43async function closeAllSequelize (servers: any[]) { 51async function closeAllSequelize (servers: any[]) {
44 for (let i = 1; i <= servers.length; i++) { 52 for (let i = 1; i <= servers.length; i++) {
45 if (sequelizes[ i ]) { 53 if (sequelizes[ i ]) {
@@ -51,6 +59,7 @@ async function closeAllSequelize (servers: any[]) {
51 59
52export { 60export {
53 setVideoField, 61 setVideoField,
62 setPlaylistField,
54 setActorField, 63 setActorField,
55 closeAllSequelize 64 closeAllSequelize
56} 65}
diff --git a/shared/utils/videos/video-playlists.ts b/shared/utils/videos/video-playlists.ts
index 7568852dc..4d110a131 100644
--- a/shared/utils/videos/video-playlists.ts
+++ b/shared/utils/videos/video-playlists.ts
@@ -127,7 +127,7 @@ function createVideoPlaylist (options: {
127 playlistAttrs: VideoPlaylistCreate, 127 playlistAttrs: VideoPlaylistCreate,
128 expectedStatus?: number 128 expectedStatus?: number
129}) { 129}) {
130 const path = '/api/v1/video-playlists/' 130 const path = '/api/v1/video-playlists'
131 131
132 const fields = omit(options.playlistAttrs, 'thumbnailfile') 132 const fields = omit(options.playlistAttrs, 'thumbnailfile')
133 133