diff options
author | Chocobozzz <me@florianbigard.com> | 2018-08-22 16:15:35 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2018-08-27 09:41:54 +0200 |
commit | 1297eb5db651a230474670c5da1517862fb9cc3e (patch) | |
tree | ecad4a0ceb0bb09e3c775262691ac68e9e0aca0c /server | |
parent | f6eebcb336c067e160a62020a5140d8d992ba384 (diff) | |
download | PeerTube-1297eb5db651a230474670c5da1517862fb9cc3e.tar.gz PeerTube-1297eb5db651a230474670c5da1517862fb9cc3e.tar.zst PeerTube-1297eb5db651a230474670c5da1517862fb9cc3e.zip |
Add refresh video on search
Diffstat (limited to 'server')
25 files changed, 490 insertions, 264 deletions
diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts index 9c2c7d6c1..d95e7cac9 100644 --- a/server/controllers/api/search.ts +++ b/server/controllers/api/search.ts | |||
@@ -13,8 +13,10 @@ import { | |||
13 | videosSearchSortValidator | 13 | videosSearchSortValidator |
14 | } from '../../middlewares' | 14 | } from '../../middlewares' |
15 | import { VideosSearchQuery } from '../../../shared/models/search' | 15 | import { VideosSearchQuery } from '../../../shared/models/search' |
16 | import { getOrCreateAccountAndVideoAndChannel } from '../../lib/activitypub' | 16 | import { getOrCreateVideoAndAccountAndChannel } from '../../lib/activitypub' |
17 | import { logger } from '../../helpers/logger' | 17 | import { logger } from '../../helpers/logger' |
18 | import { User } from '../../../shared/models/users' | ||
19 | import { CONFIG } from '../../initializers/constants' | ||
18 | 20 | ||
19 | const searchRouter = express.Router() | 21 | const searchRouter = express.Router() |
20 | 22 | ||
@@ -56,20 +58,30 @@ async function searchVideosDB (query: VideosSearchQuery, res: express.Response) | |||
56 | 58 | ||
57 | async function searchVideoUrl (url: string, res: express.Response) { | 59 | async function searchVideoUrl (url: string, res: express.Response) { |
58 | let video: VideoModel | 60 | let video: VideoModel |
61 | const user: User = res.locals.oauth ? res.locals.oauth.token.User : undefined | ||
59 | 62 | ||
60 | try { | 63 | // Check if we can fetch a remote video with the URL |
61 | const syncParam = { | 64 | if ( |
62 | likes: false, | 65 | CONFIG.SEARCH.REMOTE_URI.ANONYMOUS === true || |
63 | dislikes: false, | 66 | (CONFIG.SEARCH.REMOTE_URI.USERS === true && user !== undefined) |
64 | shares: false, | 67 | ) { |
65 | comments: false, | 68 | try { |
66 | thumbnail: true | 69 | const syncParam = { |
67 | } | 70 | likes: false, |
71 | dislikes: false, | ||
72 | shares: false, | ||
73 | comments: false, | ||
74 | thumbnail: true, | ||
75 | refreshVideo: false | ||
76 | } | ||
68 | 77 | ||
69 | const res = await getOrCreateAccountAndVideoAndChannel(url, syncParam) | 78 | const res = await getOrCreateVideoAndAccountAndChannel(url, syncParam) |
70 | video = res ? res.video : undefined | 79 | video = res ? res.video : undefined |
71 | } catch (err) { | 80 | } catch (err) { |
72 | logger.info('Cannot search remote video %s.', url) | 81 | logger.info('Cannot search remote video %s.', url) |
82 | } | ||
83 | } else { | ||
84 | video = await VideoModel.loadByUrlAndPopulateAccount(url) | ||
73 | } | 85 | } |
74 | 86 | ||
75 | return res.json({ | 87 | return res.json({ |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 99b10a7fc..cd709cd3f 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -181,6 +181,12 @@ const CONFIG = { | |||
181 | LOG: { | 181 | LOG: { |
182 | LEVEL: config.get<string>('log.level') | 182 | LEVEL: config.get<string>('log.level') |
183 | }, | 183 | }, |
184 | SEARCH: { | ||
185 | REMOTE_URI: { | ||
186 | USERS: config.get<boolean>('search.remote_uri.users'), | ||
187 | ANONYMOUS: config.get<boolean>('search.remote_uri.anonymous') | ||
188 | } | ||
189 | }, | ||
184 | ADMIN: { | 190 | ADMIN: { |
185 | get EMAIL () { return config.get<string>('admin.email') } | 191 | get EMAIL () { return config.get<string>('admin.email') } |
186 | }, | 192 | }, |
@@ -462,7 +468,8 @@ const ACTIVITY_PUB = { | |||
462 | MAGNET: [ 'application/x-bittorrent;x-scheme-handler/magnet' ] | 468 | MAGNET: [ 'application/x-bittorrent;x-scheme-handler/magnet' ] |
463 | }, | 469 | }, |
464 | MAX_RECURSION_COMMENTS: 100, | 470 | MAX_RECURSION_COMMENTS: 100, |
465 | ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000 // 1 day | 471 | ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000, // 1 day |
472 | VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 // 1 day | ||
466 | } | 473 | } |
467 | 474 | ||
468 | const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = { | 475 | const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = { |
@@ -574,6 +581,7 @@ if (isTestInstance() === true) { | |||
574 | 581 | ||
575 | ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2 | 582 | ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2 |
576 | ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 seconds | 583 | ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 seconds |
584 | ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL = 10 * 1000 // 10 seconds | ||
577 | 585 | ||
578 | CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB | 586 | CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB |
579 | 587 | ||
diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts index b08156aa1..814556817 100644 --- a/server/lib/activitypub/process/process-announce.ts +++ b/server/lib/activitypub/process/process-announce.ts | |||
@@ -6,7 +6,7 @@ import { VideoModel } from '../../../models/video/video' | |||
6 | import { VideoShareModel } from '../../../models/video/video-share' | 6 | import { VideoShareModel } from '../../../models/video/video-share' |
7 | import { getOrCreateActorAndServerAndModel } from '../actor' | 7 | import { getOrCreateActorAndServerAndModel } from '../actor' |
8 | import { forwardVideoRelatedActivity } from '../send/utils' | 8 | import { forwardVideoRelatedActivity } from '../send/utils' |
9 | import { getOrCreateAccountAndVideoAndChannel } from '../videos' | 9 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' |
10 | 10 | ||
11 | async function processAnnounceActivity (activity: ActivityAnnounce) { | 11 | async function processAnnounceActivity (activity: ActivityAnnounce) { |
12 | const actorAnnouncer = await getOrCreateActorAndServerAndModel(activity.actor) | 12 | const actorAnnouncer = await getOrCreateActorAndServerAndModel(activity.actor) |
@@ -25,7 +25,7 @@ export { | |||
25 | async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { | 25 | async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { |
26 | const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id | 26 | const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id |
27 | 27 | ||
28 | const { video } = await getOrCreateAccountAndVideoAndChannel(objectUri) | 28 | const { video } = await getOrCreateVideoAndAccountAndChannel(objectUri) |
29 | 29 | ||
30 | return sequelizeTypescript.transaction(async t => { | 30 | return sequelizeTypescript.transaction(async t => { |
31 | // Add share entry | 31 | // Add share entry |
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 9655d015f..e8f5ade06 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts | |||
@@ -10,7 +10,7 @@ import { VideoAbuseModel } from '../../../models/video/video-abuse' | |||
10 | import { VideoCommentModel } from '../../../models/video/video-comment' | 10 | import { VideoCommentModel } from '../../../models/video/video-comment' |
11 | import { getOrCreateActorAndServerAndModel } from '../actor' | 11 | import { getOrCreateActorAndServerAndModel } from '../actor' |
12 | import { resolveThread } from '../video-comments' | 12 | import { resolveThread } from '../video-comments' |
13 | import { getOrCreateAccountAndVideoAndChannel } from '../videos' | 13 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' |
14 | import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils' | 14 | import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils' |
15 | 15 | ||
16 | async function processCreateActivity (activity: ActivityCreate) { | 16 | async function processCreateActivity (activity: ActivityCreate) { |
@@ -45,7 +45,7 @@ export { | |||
45 | async function processCreateVideo (activity: ActivityCreate) { | 45 | async function processCreateVideo (activity: ActivityCreate) { |
46 | const videoToCreateData = activity.object as VideoTorrentObject | 46 | const videoToCreateData = activity.object as VideoTorrentObject |
47 | 47 | ||
48 | const { video } = await getOrCreateAccountAndVideoAndChannel(videoToCreateData) | 48 | const { video } = await getOrCreateVideoAndAccountAndChannel(videoToCreateData) |
49 | 49 | ||
50 | return video | 50 | return video |
51 | } | 51 | } |
@@ -56,7 +56,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea | |||
56 | 56 | ||
57 | if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) | 57 | if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) |
58 | 58 | ||
59 | const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object) | 59 | const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object) |
60 | 60 | ||
61 | return sequelizeTypescript.transaction(async t => { | 61 | return sequelizeTypescript.transaction(async t => { |
62 | const rate = { | 62 | const rate = { |
@@ -83,7 +83,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea | |||
83 | async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { | 83 | async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { |
84 | const view = activity.object as ViewObject | 84 | const view = activity.object as ViewObject |
85 | 85 | ||
86 | const { video } = await getOrCreateAccountAndVideoAndChannel(view.object) | 86 | const { video } = await getOrCreateVideoAndAccountAndChannel(view.object) |
87 | 87 | ||
88 | const actor = await ActorModel.loadByUrl(view.actor) | 88 | const actor = await ActorModel.loadByUrl(view.actor) |
89 | if (!actor) throw new Error('Unknown actor ' + view.actor) | 89 | if (!actor) throw new Error('Unknown actor ' + view.actor) |
@@ -103,7 +103,7 @@ async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateDat | |||
103 | const account = actor.Account | 103 | const account = actor.Account |
104 | if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url) | 104 | if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url) |
105 | 105 | ||
106 | const { video } = await getOrCreateAccountAndVideoAndChannel(videoAbuseToCreateData.object) | 106 | const { video } = await getOrCreateVideoAndAccountAndChannel(videoAbuseToCreateData.object) |
107 | 107 | ||
108 | return sequelizeTypescript.transaction(async t => { | 108 | return sequelizeTypescript.transaction(async t => { |
109 | const videoAbuseData = { | 109 | const videoAbuseData = { |
diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts index d0865b78c..9e1664fd8 100644 --- a/server/lib/activitypub/process/process-like.ts +++ b/server/lib/activitypub/process/process-like.ts | |||
@@ -5,7 +5,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat | |||
5 | import { ActorModel } from '../../../models/activitypub/actor' | 5 | import { ActorModel } from '../../../models/activitypub/actor' |
6 | import { getOrCreateActorAndServerAndModel } from '../actor' | 6 | import { getOrCreateActorAndServerAndModel } from '../actor' |
7 | import { forwardVideoRelatedActivity } from '../send/utils' | 7 | import { forwardVideoRelatedActivity } from '../send/utils' |
8 | import { getOrCreateAccountAndVideoAndChannel } from '../videos' | 8 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' |
9 | 9 | ||
10 | async function processLikeActivity (activity: ActivityLike) { | 10 | async function processLikeActivity (activity: ActivityLike) { |
11 | const actor = await getOrCreateActorAndServerAndModel(activity.actor) | 11 | const actor = await getOrCreateActorAndServerAndModel(activity.actor) |
@@ -27,7 +27,7 @@ async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) { | |||
27 | const byAccount = byActor.Account | 27 | const byAccount = byActor.Account |
28 | if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) | 28 | if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) |
29 | 29 | ||
30 | const { video } = await getOrCreateAccountAndVideoAndChannel(videoUrl) | 30 | const { video } = await getOrCreateVideoAndAccountAndChannel(videoUrl) |
31 | 31 | ||
32 | return sequelizeTypescript.transaction(async t => { | 32 | return sequelizeTypescript.transaction(async t => { |
33 | const rate = { | 33 | const rate = { |
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index b6de107ad..eab9e3d61 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts | |||
@@ -9,7 +9,7 @@ import { AccountVideoRateModel } from '../../../models/account/account-video-rat | |||
9 | import { ActorModel } from '../../../models/activitypub/actor' | 9 | import { ActorModel } from '../../../models/activitypub/actor' |
10 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 10 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' |
11 | import { forwardVideoRelatedActivity } from '../send/utils' | 11 | import { forwardVideoRelatedActivity } from '../send/utils' |
12 | import { getOrCreateAccountAndVideoAndChannel } from '../videos' | 12 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' |
13 | import { VideoShareModel } from '../../../models/video/video-share' | 13 | import { VideoShareModel } from '../../../models/video/video-share' |
14 | 14 | ||
15 | async function processUndoActivity (activity: ActivityUndo) { | 15 | async function processUndoActivity (activity: ActivityUndo) { |
@@ -43,7 +43,7 @@ export { | |||
43 | async function processUndoLike (actorUrl: string, activity: ActivityUndo) { | 43 | async function processUndoLike (actorUrl: string, activity: ActivityUndo) { |
44 | const likeActivity = activity.object as ActivityLike | 44 | const likeActivity = activity.object as ActivityLike |
45 | 45 | ||
46 | const { video } = await getOrCreateAccountAndVideoAndChannel(likeActivity.object) | 46 | const { video } = await getOrCreateVideoAndAccountAndChannel(likeActivity.object) |
47 | 47 | ||
48 | return sequelizeTypescript.transaction(async t => { | 48 | return sequelizeTypescript.transaction(async t => { |
49 | const byAccount = await AccountModel.loadByUrl(actorUrl, t) | 49 | const byAccount = await AccountModel.loadByUrl(actorUrl, t) |
@@ -67,7 +67,7 @@ async function processUndoLike (actorUrl: string, activity: ActivityUndo) { | |||
67 | async function processUndoDislike (actorUrl: string, activity: ActivityUndo) { | 67 | async function processUndoDislike (actorUrl: string, activity: ActivityUndo) { |
68 | const dislike = activity.object.object as DislikeObject | 68 | const dislike = activity.object.object as DislikeObject |
69 | 69 | ||
70 | const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object) | 70 | const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object) |
71 | 71 | ||
72 | return sequelizeTypescript.transaction(async t => { | 72 | return sequelizeTypescript.transaction(async t => { |
73 | const byAccount = await AccountModel.loadByUrl(actorUrl, t) | 73 | const byAccount = await AccountModel.loadByUrl(actorUrl, t) |
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index 11226e275..07a5ff92f 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts | |||
@@ -1,4 +1,3 @@ | |||
1 | import * as Bluebird from 'bluebird' | ||
2 | import { ActivityUpdate, VideoTorrentObject } from '../../../../shared/models/activitypub' | 1 | import { ActivityUpdate, VideoTorrentObject } from '../../../../shared/models/activitypub' |
3 | import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor' | 2 | import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor' |
4 | import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils' | 3 | import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils' |
@@ -6,19 +5,10 @@ import { logger } from '../../../helpers/logger' | |||
6 | import { sequelizeTypescript } from '../../../initializers' | 5 | import { sequelizeTypescript } from '../../../initializers' |
7 | import { AccountModel } from '../../../models/account/account' | 6 | import { AccountModel } from '../../../models/account/account' |
8 | import { ActorModel } from '../../../models/activitypub/actor' | 7 | import { ActorModel } from '../../../models/activitypub/actor' |
9 | import { TagModel } from '../../../models/video/tag' | ||
10 | import { VideoChannelModel } from '../../../models/video/video-channel' | 8 | import { VideoChannelModel } from '../../../models/video/video-channel' |
11 | import { VideoFileModel } from '../../../models/video/video-file' | ||
12 | import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor' | 9 | import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor' |
13 | import { | 10 | import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannel, updateVideoFromAP } from '../videos' |
14 | generateThumbnailFromUrl, | ||
15 | getOrCreateAccountAndVideoAndChannel, | ||
16 | getOrCreateVideoChannel, | ||
17 | videoActivityObjectToDBAttributes, | ||
18 | videoFileActivityUrlToDBAttributes | ||
19 | } from '../videos' | ||
20 | import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' | 11 | import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' |
21 | import { VideoCaptionModel } from '../../../models/video/video-caption' | ||
22 | 12 | ||
23 | async function processUpdateActivity (activity: ActivityUpdate) { | 13 | async function processUpdateActivity (activity: ActivityUpdate) { |
24 | const actor = await getOrCreateActorAndServerAndModel(activity.actor) | 14 | const actor = await getOrCreateActorAndServerAndModel(activity.actor) |
@@ -49,91 +39,10 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) | |||
49 | return undefined | 39 | return undefined |
50 | } | 40 | } |
51 | 41 | ||
52 | const res = await getOrCreateAccountAndVideoAndChannel(videoObject.id) | 42 | const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id) |
43 | const channelActor = await getOrCreateVideoChannel(videoObject) | ||
53 | 44 | ||
54 | // Fetch video channel outside the transaction | 45 | return updateVideoFromAP(video, videoObject, actor, channelActor, activity.to) |
55 | const newVideoChannelActor = await getOrCreateVideoChannel(videoObject) | ||
56 | const newVideoChannel = newVideoChannelActor.VideoChannel | ||
57 | |||
58 | logger.debug('Updating remote video "%s".', videoObject.uuid) | ||
59 | let videoInstance = res.video | ||
60 | let videoFieldsSave: any | ||
61 | |||
62 | try { | ||
63 | await sequelizeTypescript.transaction(async t => { | ||
64 | const sequelizeOptions = { | ||
65 | transaction: t | ||
66 | } | ||
67 | |||
68 | videoFieldsSave = videoInstance.toJSON() | ||
69 | |||
70 | // Check actor has the right to update the video | ||
71 | const videoChannel = videoInstance.VideoChannel | ||
72 | if (videoChannel.Account.Actor.id !== actor.id) { | ||
73 | throw new Error('Account ' + actor.url + ' does not own video channel ' + videoChannel.Actor.url) | ||
74 | } | ||
75 | |||
76 | const videoData = await videoActivityObjectToDBAttributes(newVideoChannel, videoObject, activity.to) | ||
77 | videoInstance.set('name', videoData.name) | ||
78 | videoInstance.set('uuid', videoData.uuid) | ||
79 | videoInstance.set('url', videoData.url) | ||
80 | videoInstance.set('category', videoData.category) | ||
81 | videoInstance.set('licence', videoData.licence) | ||
82 | videoInstance.set('language', videoData.language) | ||
83 | videoInstance.set('description', videoData.description) | ||
84 | videoInstance.set('support', videoData.support) | ||
85 | videoInstance.set('nsfw', videoData.nsfw) | ||
86 | videoInstance.set('commentsEnabled', videoData.commentsEnabled) | ||
87 | videoInstance.set('waitTranscoding', videoData.waitTranscoding) | ||
88 | videoInstance.set('state', videoData.state) | ||
89 | videoInstance.set('duration', videoData.duration) | ||
90 | videoInstance.set('createdAt', videoData.createdAt) | ||
91 | videoInstance.set('updatedAt', videoData.updatedAt) | ||
92 | videoInstance.set('views', videoData.views) | ||
93 | videoInstance.set('privacy', videoData.privacy) | ||
94 | videoInstance.set('channelId', videoData.channelId) | ||
95 | |||
96 | await videoInstance.save(sequelizeOptions) | ||
97 | |||
98 | // Don't block on request | ||
99 | generateThumbnailFromUrl(videoInstance, videoObject.icon) | ||
100 | .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) | ||
101 | |||
102 | // Remove old video files | ||
103 | const videoFileDestroyTasks: Bluebird<void>[] = [] | ||
104 | for (const videoFile of videoInstance.VideoFiles) { | ||
105 | videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) | ||
106 | } | ||
107 | await Promise.all(videoFileDestroyTasks) | ||
108 | |||
109 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoInstance, videoObject) | ||
110 | const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions)) | ||
111 | await Promise.all(tasks) | ||
112 | |||
113 | // Update Tags | ||
114 | const tags = videoObject.tag.map(tag => tag.name) | ||
115 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | ||
116 | await videoInstance.$set('Tags', tagInstances, sequelizeOptions) | ||
117 | |||
118 | // Update captions | ||
119 | await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(videoInstance.id, t) | ||
120 | |||
121 | const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { | ||
122 | return VideoCaptionModel.insertOrReplaceLanguage(videoInstance.id, c.identifier, t) | ||
123 | }) | ||
124 | await Promise.all(videoCaptionsPromises) | ||
125 | }) | ||
126 | |||
127 | logger.info('Remote video with uuid %s updated', videoObject.uuid) | ||
128 | } catch (err) { | ||
129 | if (videoInstance !== undefined && videoFieldsSave !== undefined) { | ||
130 | resetSequelizeInstance(videoInstance, videoFieldsSave) | ||
131 | } | ||
132 | |||
133 | // This is just a debug because we will retry the insert | ||
134 | logger.debug('Cannot update the remote video.', { err }) | ||
135 | throw err | ||
136 | } | ||
137 | } | 46 | } |
138 | 47 | ||
139 | async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) { | 48 | async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) { |
diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index 698414867..fe3d73e9b 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts | |||
@@ -6,6 +6,11 @@ import { VideoShareModel } from '../../models/video/video-share' | |||
6 | import { sendUndoAnnounce, sendVideoAnnounce } from './send' | 6 | import { sendUndoAnnounce, sendVideoAnnounce } from './send' |
7 | import { getAnnounceActivityPubUrl } from './url' | 7 | import { getAnnounceActivityPubUrl } from './url' |
8 | import { VideoChannelModel } from '../../models/video/video-channel' | 8 | import { VideoChannelModel } from '../../models/video/video-channel' |
9 | import * as Bluebird from 'bluebird' | ||
10 | import { doRequest } from '../../helpers/requests' | ||
11 | import { getOrCreateActorAndServerAndModel } from './actor' | ||
12 | import { logger } from '../../helpers/logger' | ||
13 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' | ||
9 | 14 | ||
10 | async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) { | 15 | async function shareVideoByServerAndChannel (video: VideoModel, t: Transaction) { |
11 | if (video.privacy === VideoPrivacy.PRIVATE) return undefined | 16 | if (video.privacy === VideoPrivacy.PRIVATE) return undefined |
@@ -22,8 +27,41 @@ async function changeVideoChannelShare (video: VideoModel, oldVideoChannel: Vide | |||
22 | await shareByVideoChannel(video, t) | 27 | await shareByVideoChannel(video, t) |
23 | } | 28 | } |
24 | 29 | ||
30 | async function addVideoShares (shareUrls: string[], instance: VideoModel) { | ||
31 | await Bluebird.map(shareUrls, async shareUrl => { | ||
32 | try { | ||
33 | // Fetch url | ||
34 | const { body } = await doRequest({ | ||
35 | uri: shareUrl, | ||
36 | json: true, | ||
37 | activityPub: true | ||
38 | }) | ||
39 | if (!body || !body.actor) throw new Error('Body of body actor is invalid') | ||
40 | |||
41 | const actorUrl = body.actor | ||
42 | const actor = await getOrCreateActorAndServerAndModel(actorUrl) | ||
43 | |||
44 | const entry = { | ||
45 | actorId: actor.id, | ||
46 | videoId: instance.id, | ||
47 | url: shareUrl | ||
48 | } | ||
49 | |||
50 | await VideoShareModel.findOrCreate({ | ||
51 | where: { | ||
52 | url: shareUrl | ||
53 | }, | ||
54 | defaults: entry | ||
55 | }) | ||
56 | } catch (err) { | ||
57 | logger.warn('Cannot add share %s.', shareUrl, { err }) | ||
58 | } | ||
59 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | ||
60 | } | ||
61 | |||
25 | export { | 62 | export { |
26 | changeVideoChannelShare, | 63 | changeVideoChannelShare, |
64 | addVideoShares, | ||
27 | shareVideoByServerAndChannel | 65 | shareVideoByServerAndChannel |
28 | } | 66 | } |
29 | 67 | ||
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index 14c7fde69..beff557bc 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts | |||
@@ -7,7 +7,7 @@ import { ActorModel } from '../../models/activitypub/actor' | |||
7 | import { VideoModel } from '../../models/video/video' | 7 | import { VideoModel } from '../../models/video/video' |
8 | import { VideoCommentModel } from '../../models/video/video-comment' | 8 | import { VideoCommentModel } from '../../models/video/video-comment' |
9 | import { getOrCreateActorAndServerAndModel } from './actor' | 9 | import { getOrCreateActorAndServerAndModel } from './actor' |
10 | import { getOrCreateAccountAndVideoAndChannel } from './videos' | 10 | import { getOrCreateVideoAndAccountAndChannel } from './videos' |
11 | import * as Bluebird from 'bluebird' | 11 | import * as Bluebird from 'bluebird' |
12 | 12 | ||
13 | async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) { | 13 | async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) { |
@@ -91,7 +91,7 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []) { | |||
91 | 91 | ||
92 | try { | 92 | try { |
93 | // Maybe it's a reply to a video? | 93 | // Maybe it's a reply to a video? |
94 | const { video } = await getOrCreateAccountAndVideoAndChannel(url) | 94 | const { video } = await getOrCreateVideoAndAccountAndChannel(url) |
95 | 95 | ||
96 | if (comments.length !== 0) { | 96 | if (comments.length !== 0) { |
97 | const firstReply = comments[ comments.length - 1 ] | 97 | const firstReply = comments[ comments.length - 1 ] |
diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts index 19011b4ab..1619251c3 100644 --- a/server/lib/activitypub/video-rates.ts +++ b/server/lib/activitypub/video-rates.ts | |||
@@ -2,6 +2,45 @@ import { Transaction } from 'sequelize' | |||
2 | import { AccountModel } from '../../models/account/account' | 2 | import { AccountModel } from '../../models/account/account' |
3 | import { VideoModel } from '../../models/video/video' | 3 | import { VideoModel } from '../../models/video/video' |
4 | import { sendCreateDislike, sendLike, sendUndoDislike, sendUndoLike } from './send' | 4 | import { sendCreateDislike, sendLike, sendUndoDislike, sendUndoLike } from './send' |
5 | import { VideoRateType } from '../../../shared/models/videos' | ||
6 | import * as Bluebird from 'bluebird' | ||
7 | import { getOrCreateActorAndServerAndModel } from './actor' | ||
8 | import { AccountVideoRateModel } from '../../models/account/account-video-rate' | ||
9 | import { logger } from '../../helpers/logger' | ||
10 | import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers' | ||
11 | |||
12 | async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) { | ||
13 | let rateCounts = 0 | ||
14 | |||
15 | await Bluebird.map(actorUrls, async actorUrl => { | ||
16 | try { | ||
17 | const actor = await getOrCreateActorAndServerAndModel(actorUrl) | ||
18 | const [ , created ] = await AccountVideoRateModel | ||
19 | .findOrCreate({ | ||
20 | where: { | ||
21 | videoId: video.id, | ||
22 | accountId: actor.Account.id | ||
23 | }, | ||
24 | defaults: { | ||
25 | videoId: video.id, | ||
26 | accountId: actor.Account.id, | ||
27 | type: rate | ||
28 | } | ||
29 | }) | ||
30 | |||
31 | if (created) rateCounts += 1 | ||
32 | } catch (err) { | ||
33 | logger.warn('Cannot add rate %s for actor %s.', rate, actorUrl, { err }) | ||
34 | } | ||
35 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | ||
36 | |||
37 | logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid) | ||
38 | |||
39 | // This is "likes" and "dislikes" | ||
40 | if (rateCounts !== 0) await video.increment(rate + 's', { by: rateCounts }) | ||
41 | |||
42 | return | ||
43 | } | ||
5 | 44 | ||
6 | async function sendVideoRateChange (account: AccountModel, | 45 | async function sendVideoRateChange (account: AccountModel, |
7 | video: VideoModel, | 46 | video: VideoModel, |
@@ -24,5 +63,6 @@ async function sendVideoRateChange (account: AccountModel, | |||
24 | } | 63 | } |
25 | 64 | ||
26 | export { | 65 | export { |
66 | createRates, | ||
27 | sendVideoRateChange | 67 | sendVideoRateChange |
28 | } | 68 | } |
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index fac1d3fc7..388c31fe5 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts | |||
@@ -5,29 +5,30 @@ import { join } from 'path' | |||
5 | import * as request from 'request' | 5 | import * as request from 'request' |
6 | import { ActivityIconObject, VideoState } from '../../../shared/index' | 6 | import { ActivityIconObject, VideoState } from '../../../shared/index' |
7 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' | 7 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' |
8 | import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos' | 8 | import { VideoPrivacy } from '../../../shared/models/videos' |
9 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' | 9 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' |
10 | import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' | 10 | import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' |
11 | import { retryTransactionWrapper } from '../../helpers/database-utils' | 11 | import { resetSequelizeInstance, retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' |
12 | import { logger } from '../../helpers/logger' | 12 | import { logger } from '../../helpers/logger' |
13 | import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' | 13 | import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' |
14 | import { ACTIVITY_PUB, CONFIG, CRAWL_REQUEST_CONCURRENCY, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers' | 14 | import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers' |
15 | import { AccountVideoRateModel } from '../../models/account/account-video-rate' | ||
16 | import { ActorModel } from '../../models/activitypub/actor' | 15 | import { ActorModel } from '../../models/activitypub/actor' |
17 | import { TagModel } from '../../models/video/tag' | 16 | import { TagModel } from '../../models/video/tag' |
18 | import { VideoModel } from '../../models/video/video' | 17 | import { VideoModel } from '../../models/video/video' |
19 | import { VideoChannelModel } from '../../models/video/video-channel' | 18 | import { VideoChannelModel } from '../../models/video/video-channel' |
20 | import { VideoFileModel } from '../../models/video/video-file' | 19 | import { VideoFileModel } from '../../models/video/video-file' |
21 | import { VideoShareModel } from '../../models/video/video-share' | 20 | import { getOrCreateActorAndServerAndModel, updateActorAvatarInstance } from './actor' |
22 | import { getOrCreateActorAndServerAndModel } from './actor' | ||
23 | import { addVideoComments } from './video-comments' | 21 | import { addVideoComments } from './video-comments' |
24 | import { crawlCollectionPage } from './crawl' | 22 | import { crawlCollectionPage } from './crawl' |
25 | import { sendCreateVideo, sendUpdateVideo } from './send' | 23 | import { sendCreateVideo, sendUpdateVideo } from './send' |
26 | import { shareVideoByServerAndChannel } from './index' | ||
27 | import { isArray } from '../../helpers/custom-validators/misc' | 24 | import { isArray } from '../../helpers/custom-validators/misc' |
28 | import { VideoCaptionModel } from '../../models/video/video-caption' | 25 | import { VideoCaptionModel } from '../../models/video/video-caption' |
29 | import { JobQueue } from '../job-queue' | 26 | import { JobQueue } from '../job-queue' |
30 | import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher' | 27 | import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher' |
28 | import { getUrlFromWebfinger } from '../../helpers/webfinger' | ||
29 | import { createRates } from './video-rates' | ||
30 | import { addVideoShares, shareVideoByServerAndChannel } from './share' | ||
31 | import { AccountModel } from '../../models/account/account' | ||
31 | 32 | ||
32 | async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { | 33 | async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { |
33 | // If the video is not private and published, we federate it | 34 | // If the video is not private and published, we federate it |
@@ -180,15 +181,11 @@ function getOrCreateVideoChannel (videoObject: VideoTorrentObject) { | |||
180 | return getOrCreateActorAndServerAndModel(channel.id) | 181 | return getOrCreateActorAndServerAndModel(channel.id) |
181 | } | 182 | } |
182 | 183 | ||
183 | async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { | 184 | async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { |
184 | logger.debug('Adding remote video %s.', videoObject.id) | 185 | logger.debug('Adding remote video %s.', videoObject.id) |
185 | 186 | ||
186 | const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { | 187 | const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { |
187 | const sequelizeOptions = { | 188 | const sequelizeOptions = { transaction: t } |
188 | transaction: t | ||
189 | } | ||
190 | const videoFromDatabase = await VideoModel.loadByUUIDOrURLAndPopulateAccount(videoObject.uuid, videoObject.id, t) | ||
191 | if (videoFromDatabase) return videoFromDatabase | ||
192 | 189 | ||
193 | const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) | 190 | const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) |
194 | const video = VideoModel.build(videoData) | 191 | const video = VideoModel.build(videoData) |
@@ -230,26 +227,32 @@ async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor: | |||
230 | } | 227 | } |
231 | 228 | ||
232 | type SyncParam = { | 229 | type SyncParam = { |
233 | likes: boolean, | 230 | likes: boolean |
234 | dislikes: boolean, | 231 | dislikes: boolean |
235 | shares: boolean, | 232 | shares: boolean |
236 | comments: boolean, | 233 | comments: boolean |
237 | thumbnail: boolean | 234 | thumbnail: boolean |
235 | refreshVideo: boolean | ||
238 | } | 236 | } |
239 | async function getOrCreateAccountAndVideoAndChannel ( | 237 | async function getOrCreateVideoAndAccountAndChannel ( |
240 | videoObject: VideoTorrentObject | string, | 238 | videoObject: VideoTorrentObject | string, |
241 | syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true } | 239 | syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } |
242 | ) { | 240 | ) { |
243 | const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id | 241 | const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id |
244 | 242 | ||
245 | const videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl) | 243 | let videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl) |
246 | if (videoFromDatabase) return { video: videoFromDatabase } | 244 | if (videoFromDatabase) { |
245 | const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase) | ||
246 | if (syncParam.refreshVideo === true) videoFromDatabase = await p | ||
247 | |||
248 | return { video: videoFromDatabase } | ||
249 | } | ||
247 | 250 | ||
248 | const fetchedVideo = await fetchRemoteVideo(videoUrl) | 251 | const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) |
249 | if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) | 252 | if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) |
250 | 253 | ||
251 | const channelActor = await getOrCreateVideoChannel(fetchedVideo) | 254 | const channelActor = await getOrCreateVideoChannel(fetchedVideo) |
252 | const video = await retryTransactionWrapper(getOrCreateVideo, fetchedVideo, channelActor, syncParam.thumbnail) | 255 | const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail) |
253 | 256 | ||
254 | // Process outside the transaction because we could fetch remote data | 257 | // Process outside the transaction because we could fetch remote data |
255 | 258 | ||
@@ -290,101 +293,153 @@ async function getOrCreateAccountAndVideoAndChannel ( | |||
290 | return { video } | 293 | return { video } |
291 | } | 294 | } |
292 | 295 | ||
293 | async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) { | 296 | async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> { |
294 | let rateCounts = 0 | 297 | const options = { |
295 | 298 | uri: videoUrl, | |
296 | await Bluebird.map(actorUrls, async actorUrl => { | 299 | method: 'GET', |
297 | try { | 300 | json: true, |
298 | const actor = await getOrCreateActorAndServerAndModel(actorUrl) | 301 | activityPub: true |
299 | const [ , created ] = await AccountVideoRateModel | 302 | } |
300 | .findOrCreate({ | 303 | |
301 | where: { | 304 | logger.info('Fetching remote video %s.', videoUrl) |
302 | videoId: video.id, | 305 | |
303 | accountId: actor.Account.id | 306 | const { response, body } = await doRequest(options) |
304 | }, | 307 | |
305 | defaults: { | 308 | if (sanitizeAndCheckVideoTorrentObject(body) === false) { |
306 | videoId: video.id, | 309 | logger.debug('Remote video JSON is not valid.', { body }) |
307 | accountId: actor.Account.id, | 310 | return { response, videoObject: undefined } |
308 | type: rate | 311 | } |
309 | } | 312 | |
310 | }) | 313 | return { response, videoObject: body } |
311 | 314 | } | |
312 | if (created) rateCounts += 1 | 315 | |
313 | } catch (err) { | 316 | async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> { |
314 | logger.warn('Cannot add rate %s for actor %s.', rate, actorUrl, { err }) | 317 | if (!video.isOutdated()) return video |
318 | |||
319 | try { | ||
320 | const { response, videoObject } = await fetchRemoteVideo(video.url) | ||
321 | if (response.statusCode === 404) { | ||
322 | // Video does not exist anymore | ||
323 | await video.destroy() | ||
324 | return undefined | ||
315 | } | 325 | } |
316 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | ||
317 | 326 | ||
318 | logger.info('Adding %d %s to video %s.', rateCounts, rate, video.uuid) | 327 | if (videoObject === undefined) { |
328 | logger.warn('Cannot refresh remote video: invalid body.') | ||
329 | return video | ||
330 | } | ||
319 | 331 | ||
320 | // This is "likes" and "dislikes" | 332 | const channelActor = await getOrCreateVideoChannel(videoObject) |
321 | if (rateCounts !== 0) await video.increment(rate + 's', { by: rateCounts }) | 333 | const account = await AccountModel.load(channelActor.VideoChannel.accountId) |
334 | return updateVideoFromAP(video, videoObject, account.Actor, channelActor) | ||
322 | 335 | ||
323 | return | 336 | } catch (err) { |
337 | logger.warn('Cannot refresh video.', { err }) | ||
338 | return video | ||
339 | } | ||
324 | } | 340 | } |
325 | 341 | ||
326 | async function addVideoShares (shareUrls: string[], instance: VideoModel) { | 342 | async function updateVideoFromAP ( |
327 | await Bluebird.map(shareUrls, async shareUrl => { | 343 | video: VideoModel, |
328 | try { | 344 | videoObject: VideoTorrentObject, |
329 | // Fetch url | 345 | accountActor: ActorModel, |
330 | const { body } = await doRequest({ | 346 | channelActor: ActorModel, |
331 | uri: shareUrl, | 347 | overrideTo?: string[] |
332 | json: true, | 348 | ) { |
333 | activityPub: true | 349 | logger.debug('Updating remote video "%s".', videoObject.uuid) |
334 | }) | 350 | let videoFieldsSave: any |
335 | if (!body || !body.actor) throw new Error('Body of body actor is invalid') | 351 | |
352 | try { | ||
353 | const updatedVideo: VideoModel = await sequelizeTypescript.transaction(async t => { | ||
354 | const sequelizeOptions = { | ||
355 | transaction: t | ||
356 | } | ||
336 | 357 | ||
337 | const actorUrl = body.actor | 358 | videoFieldsSave = video.toJSON() |
338 | const actor = await getOrCreateActorAndServerAndModel(actorUrl) | ||
339 | 359 | ||
340 | const entry = { | 360 | // Check actor has the right to update the video |
341 | actorId: actor.id, | 361 | const videoChannel = video.VideoChannel |
342 | videoId: instance.id, | 362 | if (videoChannel.Account.Actor.id !== accountActor.id) { |
343 | url: shareUrl | 363 | throw new Error('Account ' + accountActor.url + ' does not own video channel ' + videoChannel.Actor.url) |
344 | } | 364 | } |
345 | 365 | ||
346 | await VideoShareModel.findOrCreate({ | 366 | const to = overrideTo ? overrideTo : videoObject.to |
347 | where: { | 367 | const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, to) |
348 | url: shareUrl | 368 | video.set('name', videoData.name) |
349 | }, | 369 | video.set('uuid', videoData.uuid) |
350 | defaults: entry | 370 | video.set('url', videoData.url) |
351 | }) | 371 | video.set('category', videoData.category) |
352 | } catch (err) { | 372 | video.set('licence', videoData.licence) |
353 | logger.warn('Cannot add share %s.', shareUrl, { err }) | 373 | video.set('language', videoData.language) |
354 | } | 374 | video.set('description', videoData.description) |
355 | }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) | 375 | video.set('support', videoData.support) |
356 | } | 376 | video.set('nsfw', videoData.nsfw) |
377 | video.set('commentsEnabled', videoData.commentsEnabled) | ||
378 | video.set('waitTranscoding', videoData.waitTranscoding) | ||
379 | video.set('state', videoData.state) | ||
380 | video.set('duration', videoData.duration) | ||
381 | video.set('createdAt', videoData.createdAt) | ||
382 | video.set('publishedAt', videoData.publishedAt) | ||
383 | video.set('views', videoData.views) | ||
384 | video.set('privacy', videoData.privacy) | ||
385 | video.set('channelId', videoData.channelId) | ||
386 | |||
387 | await video.save(sequelizeOptions) | ||
388 | |||
389 | // Don't block on request | ||
390 | generateThumbnailFromUrl(video, videoObject.icon) | ||
391 | .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) | ||
392 | |||
393 | // Remove old video files | ||
394 | const videoFileDestroyTasks: Bluebird<void>[] = [] | ||
395 | for (const videoFile of video.VideoFiles) { | ||
396 | videoFileDestroyTasks.push(videoFile.destroy(sequelizeOptions)) | ||
397 | } | ||
398 | await Promise.all(videoFileDestroyTasks) | ||
357 | 399 | ||
358 | async function fetchRemoteVideo (videoUrl: string): Promise<VideoTorrentObject> { | 400 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(video, videoObject) |
359 | const options = { | 401 | const tasks = videoFileAttributes.map(f => VideoFileModel.create(f, sequelizeOptions)) |
360 | uri: videoUrl, | 402 | await Promise.all(tasks) |
361 | method: 'GET', | ||
362 | json: true, | ||
363 | activityPub: true | ||
364 | } | ||
365 | 403 | ||
366 | logger.info('Fetching remote video %s.', videoUrl) | 404 | // Update Tags |
405 | const tags = videoObject.tag.map(tag => tag.name) | ||
406 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | ||
407 | await video.$set('Tags', tagInstances, sequelizeOptions) | ||
367 | 408 | ||
368 | const { body } = await doRequest(options) | 409 | // Update captions |
410 | await VideoCaptionModel.deleteAllCaptionsOfRemoteVideo(video.id, t) | ||
369 | 411 | ||
370 | if (sanitizeAndCheckVideoTorrentObject(body) === false) { | 412 | const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { |
371 | logger.debug('Remote video JSON is not valid.', { body }) | 413 | return VideoCaptionModel.insertOrReplaceLanguage(video.id, c.identifier, t) |
372 | return undefined | 414 | }) |
373 | } | 415 | await Promise.all(videoCaptionsPromises) |
416 | }) | ||
417 | |||
418 | logger.info('Remote video with uuid %s updated', videoObject.uuid) | ||
374 | 419 | ||
375 | return body | 420 | return updatedVideo |
421 | } catch (err) { | ||
422 | if (video !== undefined && videoFieldsSave !== undefined) { | ||
423 | resetSequelizeInstance(video, videoFieldsSave) | ||
424 | } | ||
425 | |||
426 | // This is just a debug because we will retry the insert | ||
427 | logger.debug('Cannot update the remote video.', { err }) | ||
428 | throw err | ||
429 | } | ||
376 | } | 430 | } |
377 | 431 | ||
378 | export { | 432 | export { |
433 | updateVideoFromAP, | ||
379 | federateVideoIfNeeded, | 434 | federateVideoIfNeeded, |
380 | fetchRemoteVideo, | 435 | fetchRemoteVideo, |
381 | getOrCreateAccountAndVideoAndChannel, | 436 | getOrCreateVideoAndAccountAndChannel, |
382 | fetchRemoteVideoStaticFile, | 437 | fetchRemoteVideoStaticFile, |
383 | fetchRemoteVideoDescription, | 438 | fetchRemoteVideoDescription, |
384 | generateThumbnailFromUrl, | 439 | generateThumbnailFromUrl, |
385 | videoActivityObjectToDBAttributes, | 440 | videoActivityObjectToDBAttributes, |
386 | videoFileActivityUrlToDBAttributes, | 441 | videoFileActivityUrlToDBAttributes, |
387 | getOrCreateVideo, | 442 | createVideo, |
388 | getOrCreateVideoChannel, | 443 | getOrCreateVideoChannel, |
389 | addVideoShares, | 444 | addVideoShares, |
390 | createRates | 445 | createRates |
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 25a1cd177..7acbc60f7 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -56,6 +56,7 @@ import { generateImageFromVideoFile, getVideoFileFPS, getVideoFileResolution, tr | |||
56 | import { logger } from '../../helpers/logger' | 56 | import { logger } from '../../helpers/logger' |
57 | import { getServerActor } from '../../helpers/utils' | 57 | import { getServerActor } from '../../helpers/utils' |
58 | import { | 58 | import { |
59 | ACTIVITY_PUB, | ||
59 | API_VERSION, | 60 | API_VERSION, |
60 | CONFIG, | 61 | CONFIG, |
61 | CONSTRAINTS_FIELDS, | 62 | CONSTRAINTS_FIELDS, |
@@ -1004,21 +1005,6 @@ export class VideoModel extends Model<VideoModel> { | |||
1004 | return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) | 1005 | return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) |
1005 | } | 1006 | } |
1006 | 1007 | ||
1007 | static loadByUUIDOrURLAndPopulateAccount (uuid: string, url: string, t?: Sequelize.Transaction) { | ||
1008 | const query: IFindOptions<VideoModel> = { | ||
1009 | where: { | ||
1010 | [Sequelize.Op.or]: [ | ||
1011 | { uuid }, | ||
1012 | { url } | ||
1013 | ] | ||
1014 | } | ||
1015 | } | ||
1016 | |||
1017 | if (t !== undefined) query.transaction = t | ||
1018 | |||
1019 | return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query) | ||
1020 | } | ||
1021 | |||
1022 | static loadAndPopulateAccountAndServerAndTags (id: number) { | 1008 | static loadAndPopulateAccountAndServerAndTags (id: number) { |
1023 | const options = { | 1009 | const options = { |
1024 | order: [ [ 'Tags', 'name', 'ASC' ] ] | 1010 | order: [ [ 'Tags', 'name', 'ASC' ] ] |
@@ -1646,6 +1632,17 @@ export class VideoModel extends Model<VideoModel> { | |||
1646 | return 'PT' + this.duration + 'S' | 1632 | return 'PT' + this.duration + 'S' |
1647 | } | 1633 | } |
1648 | 1634 | ||
1635 | isOutdated () { | ||
1636 | if (this.isOwned()) return false | ||
1637 | |||
1638 | const now = Date.now() | ||
1639 | const createdAtTime = this.createdAt.getTime() | ||
1640 | const updatedAtTime = this.updatedAt.getTime() | ||
1641 | |||
1642 | return (now - createdAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL && | ||
1643 | (now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL | ||
1644 | } | ||
1645 | |||
1649 | private getBaseUrls () { | 1646 | private getBaseUrls () { |
1650 | let baseUrlHttp | 1647 | let baseUrlHttp |
1651 | let baseUrlWs | 1648 | let baseUrlWs |
diff --git a/server/tests/api/index-1.ts b/server/tests/api/index-1.ts new file mode 100644 index 000000000..80d752f42 --- /dev/null +++ b/server/tests/api/index-1.ts | |||
@@ -0,0 +1,2 @@ | |||
1 | import './check-params' | ||
2 | import './search' | ||
diff --git a/server/tests/api/index-2.ts b/server/tests/api/index-2.ts new file mode 100644 index 000000000..ed93faa91 --- /dev/null +++ b/server/tests/api/index-2.ts | |||
@@ -0,0 +1,2 @@ | |||
1 | import './server' | ||
2 | import './users' | ||
diff --git a/server/tests/api/index-3.ts b/server/tests/api/index-3.ts new file mode 100644 index 000000000..39823b82c --- /dev/null +++ b/server/tests/api/index-3.ts | |||
@@ -0,0 +1 @@ | |||
import './videos' | |||
diff --git a/server/tests/api/index-fast.ts b/server/tests/api/index-fast.ts deleted file mode 100644 index 02ffdd4f1..000000000 --- a/server/tests/api/index-fast.ts +++ /dev/null | |||
@@ -1,18 +0,0 @@ | |||
1 | // Order of the tests we want to execute | ||
2 | import './server/stats' | ||
3 | import './check-params' | ||
4 | import './users/users' | ||
5 | import './videos/single-server' | ||
6 | import './videos/video-abuse' | ||
7 | import './videos/video-captions' | ||
8 | import './videos/video-blacklist' | ||
9 | import './videos/video-blacklist-management' | ||
10 | import './videos/video-description' | ||
11 | import './videos/video-nsfw' | ||
12 | import './videos/video-privacy' | ||
13 | import './videos/services' | ||
14 | import './server/email' | ||
15 | import './server/config' | ||
16 | import './server/reverse-proxy' | ||
17 | import './search/search-videos' | ||
18 | import './server/tracker' | ||
diff --git a/server/tests/api/index-slow.ts b/server/tests/api/index-slow.ts deleted file mode 100644 index e24a7b664..000000000 --- a/server/tests/api/index-slow.ts +++ /dev/null | |||
@@ -1,12 +0,0 @@ | |||
1 | // Order of the tests we want to execute | ||
2 | import './videos/video-channels' | ||
3 | import './videos/video-transcoder' | ||
4 | import './videos/multiple-servers' | ||
5 | import './server/follows' | ||
6 | import './server/jobs' | ||
7 | import './videos/video-comments' | ||
8 | import './users/users-multiple-servers' | ||
9 | import './users/user-subscriptions' | ||
10 | import './server/handle-down' | ||
11 | import './videos/video-schedule-update' | ||
12 | import './videos/video-imports' | ||
diff --git a/server/tests/api/index.ts b/server/tests/api/index.ts index 258502d26..2d996dbf9 100644 --- a/server/tests/api/index.ts +++ b/server/tests/api/index.ts | |||
@@ -1,3 +1,4 @@ | |||
1 | // Order of the tests we want to execute | 1 | // Order of the tests we want to execute |
2 | import './index-fast' | 2 | import './index-1' |
3 | import './index-slow' | 3 | import './index-2' |
4 | import './index-3' | ||
diff --git a/server/tests/api/search/index.ts b/server/tests/api/search/index.ts new file mode 100644 index 000000000..64b3d0910 --- /dev/null +++ b/server/tests/api/search/index.ts | |||
@@ -0,0 +1,2 @@ | |||
1 | import './search-activitypub-videos' | ||
2 | import './search-videos' | ||
diff --git a/server/tests/api/search/search-activitypub-videos.ts b/server/tests/api/search/search-activitypub-videos.ts new file mode 100644 index 000000000..6dc792696 --- /dev/null +++ b/server/tests/api/search/search-activitypub-videos.ts | |||
@@ -0,0 +1,161 @@ | |||
1 | /* tslint:disable:no-unused-expression */ | ||
2 | |||
3 | import * as chai from 'chai' | ||
4 | import 'mocha' | ||
5 | import { | ||
6 | addVideoChannel, | ||
7 | flushAndRunMultipleServers, | ||
8 | flushTests, | ||
9 | getVideosList, | ||
10 | killallServers, | ||
11 | removeVideo, | ||
12 | searchVideoWithToken, | ||
13 | ServerInfo, | ||
14 | setAccessTokensToServers, | ||
15 | updateVideo, | ||
16 | uploadVideo, | ||
17 | wait, | ||
18 | searchVideo | ||
19 | } from '../../utils' | ||
20 | import { waitJobs } from '../../utils/server/jobs' | ||
21 | import { Video, VideoPrivacy } from '../../../../shared/models/videos' | ||
22 | |||
23 | const expect = chai.expect | ||
24 | |||
25 | describe('Test a ActivityPub videos search', function () { | ||
26 | let servers: ServerInfo[] | ||
27 | let videoServer1UUID: string | ||
28 | let videoServer2UUID: string | ||
29 | |||
30 | before(async function () { | ||
31 | this.timeout(120000) | ||
32 | |||
33 | await flushTests() | ||
34 | |||
35 | servers = await flushAndRunMultipleServers(2) | ||
36 | |||
37 | await setAccessTokensToServers(servers) | ||
38 | |||
39 | { | ||
40 | const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, { name: 'video 1 on server 1' }) | ||
41 | videoServer1UUID = res.body.video.uuid | ||
42 | } | ||
43 | |||
44 | { | ||
45 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 on server 2' }) | ||
46 | videoServer2UUID = res.body.video.uuid | ||
47 | } | ||
48 | |||
49 | await waitJobs(servers) | ||
50 | }) | ||
51 | |||
52 | it('Should not find a remote video', async function () { | ||
53 | { | ||
54 | const res = await searchVideoWithToken(servers[ 0 ].url, 'http://localhost:9002/videos/watch/43', servers[ 0 ].accessToken) | ||
55 | |||
56 | expect(res.body.total).to.equal(0) | ||
57 | expect(res.body.data).to.be.an('array') | ||
58 | expect(res.body.data).to.have.lengthOf(0) | ||
59 | } | ||
60 | |||
61 | { | ||
62 | const res = await searchVideo(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID) | ||
63 | |||
64 | expect(res.body.total).to.equal(0) | ||
65 | expect(res.body.data).to.be.an('array') | ||
66 | expect(res.body.data).to.have.lengthOf(0) | ||
67 | } | ||
68 | }) | ||
69 | |||
70 | it('Should search a local video', async function () { | ||
71 | const res = await searchVideo(servers[0].url, 'http://localhost:9001/videos/watch/' + videoServer1UUID) | ||
72 | |||
73 | expect(res.body.total).to.equal(1) | ||
74 | expect(res.body.data).to.be.an('array') | ||
75 | expect(res.body.data).to.have.lengthOf(1) | ||
76 | expect(res.body.data[0].name).to.equal('video 1 on server 1') | ||
77 | }) | ||
78 | |||
79 | it('Should search a remote video', async function () { | ||
80 | const res = await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken) | ||
81 | |||
82 | expect(res.body.total).to.equal(1) | ||
83 | expect(res.body.data).to.be.an('array') | ||
84 | expect(res.body.data).to.have.lengthOf(1) | ||
85 | expect(res.body.data[0].name).to.equal('video 1 on server 2') | ||
86 | }) | ||
87 | |||
88 | it('Should not list this remote video', async function () { | ||
89 | const res = await getVideosList(servers[0].url) | ||
90 | expect(res.body.total).to.equal(1) | ||
91 | expect(res.body.data).to.have.lengthOf(1) | ||
92 | expect(res.body.data[0].name).to.equal('video 1 on server 1') | ||
93 | }) | ||
94 | |||
95 | it('Should update video of server 2, and refresh it on server 1', async function () { | ||
96 | this.timeout(60000) | ||
97 | |||
98 | const channelAttributes = { | ||
99 | name: 'super_channel', | ||
100 | displayName: 'super channel' | ||
101 | } | ||
102 | const resChannel = await addVideoChannel(servers[1].url, servers[1].accessToken, channelAttributes) | ||
103 | const videoChannelId = resChannel.body.videoChannel.id | ||
104 | |||
105 | const attributes = { | ||
106 | name: 'updated', | ||
107 | tag: [ 'tag1', 'tag2' ], | ||
108 | privacy: VideoPrivacy.UNLISTED, | ||
109 | channelId: videoChannelId | ||
110 | } | ||
111 | await updateVideo(servers[1].url, servers[1].accessToken, videoServer2UUID, attributes) | ||
112 | |||
113 | await waitJobs(servers) | ||
114 | // Expire video | ||
115 | await wait(10000) | ||
116 | |||
117 | // Will run refresh async | ||
118 | await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken) | ||
119 | |||
120 | // Wait refresh | ||
121 | await wait(5000) | ||
122 | |||
123 | const res = await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken) | ||
124 | expect(res.body.total).to.equal(1) | ||
125 | expect(res.body.data).to.have.lengthOf(1) | ||
126 | |||
127 | const video: Video = res.body.data[0] | ||
128 | expect(video.name).to.equal('updated') | ||
129 | expect(video.channel.name).to.equal('super_channel') | ||
130 | expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED) | ||
131 | }) | ||
132 | |||
133 | it('Should delete video of server 2, and delete it on server 1', async function () { | ||
134 | this.timeout(60000) | ||
135 | |||
136 | await removeVideo(servers[1].url, servers[1].accessToken, videoServer2UUID) | ||
137 | |||
138 | await waitJobs(servers) | ||
139 | // Expire video | ||
140 | await wait(10000) | ||
141 | |||
142 | // Will run refresh async | ||
143 | await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken) | ||
144 | |||
145 | // Wait refresh | ||
146 | await wait(5000) | ||
147 | |||
148 | const res = await searchVideoWithToken(servers[0].url, 'http://localhost:9002/videos/watch/' + videoServer2UUID, servers[0].accessToken) | ||
149 | expect(res.body.total).to.equal(0) | ||
150 | expect(res.body.data).to.have.lengthOf(0) | ||
151 | }) | ||
152 | |||
153 | after(async function () { | ||
154 | killallServers(servers) | ||
155 | |||
156 | // Keep the logs if the test failed | ||
157 | if (this['ok']) { | ||
158 | await flushTests() | ||
159 | } | ||
160 | }) | ||
161 | }) | ||
diff --git a/server/tests/api/server/index.ts b/server/tests/api/server/index.ts new file mode 100644 index 000000000..eeb8b7a28 --- /dev/null +++ b/server/tests/api/server/index.ts | |||
@@ -0,0 +1,8 @@ | |||
1 | import './config' | ||
2 | import './email' | ||
3 | import './follows' | ||
4 | import './handle-down' | ||
5 | import './jobs' | ||
6 | import './reverse-proxy' | ||
7 | import './stats' | ||
8 | import './tracker' | ||
diff --git a/server/tests/api/users/index.ts b/server/tests/api/users/index.ts new file mode 100644 index 000000000..4ce87fb91 --- /dev/null +++ b/server/tests/api/users/index.ts | |||
@@ -0,0 +1,3 @@ | |||
1 | import './user-subscriptions' | ||
2 | import './users' | ||
3 | import './users-multiple-servers' | ||
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts new file mode 100644 index 000000000..9f1230767 --- /dev/null +++ b/server/tests/api/videos/index.ts | |||
@@ -0,0 +1,15 @@ | |||
1 | import './multiple-servers' | ||
2 | import './services' | ||
3 | import './single-server' | ||
4 | import './video-abuse' | ||
5 | import './video-blacklist' | ||
6 | import './video-blacklist-management' | ||
7 | import './video-captions' | ||
8 | import './video-channels' | ||
9 | import './video-comme' | ||
10 | import './video-description' | ||
11 | import './video-impo' | ||
12 | import './video-nsfw' | ||
13 | import './video-privacy' | ||
14 | import './video-schedule-update' | ||
15 | import './video-transcoder' | ||
diff --git a/server/tests/feeds/index.ts b/server/tests/feeds/index.ts new file mode 100644 index 000000000..aa6236a91 --- /dev/null +++ b/server/tests/feeds/index.ts | |||
@@ -0,0 +1 @@ | |||
import './feeds' | |||
diff --git a/server/tests/index.ts b/server/tests/index.ts index 755fb2604..e659fd3df 100644 --- a/server/tests/index.ts +++ b/server/tests/index.ts | |||
@@ -1,5 +1,6 @@ | |||
1 | // Order of the tests we want to execute | 1 | // Order of the tests we want to execute |
2 | import './client' | 2 | import './client' |
3 | import './activitypub' | 3 | import './activitypub' |
4 | import './api/' | 4 | import './feeds/' |
5 | import './cli/' | 5 | import './cli/' |
6 | import './api/' | ||