diff options
Diffstat (limited to 'server/lib/activitypub')
-rw-r--r-- | server/lib/activitypub/process/process-announce.ts | 2 | ||||
-rw-r--r-- | server/lib/activitypub/process/process-create.ts | 10 | ||||
-rw-r--r-- | server/lib/activitypub/process/process-like.ts | 2 | ||||
-rw-r--r-- | server/lib/activitypub/process/process-undo.ts | 6 | ||||
-rw-r--r-- | server/lib/activitypub/process/process-update.ts | 4 | ||||
-rw-r--r-- | server/lib/activitypub/video-comments.ts | 2 | ||||
-rw-r--r-- | server/lib/activitypub/videos.ts | 420 |
7 files changed, 227 insertions, 219 deletions
diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts index 814556817..b968389b3 100644 --- a/server/lib/activitypub/process/process-announce.ts +++ b/server/lib/activitypub/process/process-announce.ts | |||
@@ -25,7 +25,7 @@ export { | |||
25 | async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { | 25 | async function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnounce) { |
26 | const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id | 26 | const objectUri = typeof activity.object === 'string' ? activity.object : activity.object.id |
27 | 27 | ||
28 | const { video } = await getOrCreateVideoAndAccountAndChannel(objectUri) | 28 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: objectUri }) |
29 | 29 | ||
30 | return sequelizeTypescript.transaction(async t => { | 30 | return sequelizeTypescript.transaction(async t => { |
31 | // Add share entry | 31 | // Add share entry |
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 32e555acf..99841da14 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts | |||
@@ -48,7 +48,7 @@ export { | |||
48 | async function processCreateVideo (activity: ActivityCreate) { | 48 | async function processCreateVideo (activity: ActivityCreate) { |
49 | const videoToCreateData = activity.object as VideoTorrentObject | 49 | const videoToCreateData = activity.object as VideoTorrentObject |
50 | 50 | ||
51 | const { video } = await getOrCreateVideoAndAccountAndChannel(videoToCreateData) | 51 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoToCreateData }) |
52 | 52 | ||
53 | return video | 53 | return video |
54 | } | 54 | } |
@@ -59,7 +59,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea | |||
59 | 59 | ||
60 | if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) | 60 | if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) |
61 | 61 | ||
62 | const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object) | 62 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) |
63 | 63 | ||
64 | return sequelizeTypescript.transaction(async t => { | 64 | return sequelizeTypescript.transaction(async t => { |
65 | const rate = { | 65 | const rate = { |
@@ -86,7 +86,7 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea | |||
86 | async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { | 86 | async function processCreateView (byActor: ActorModel, activity: ActivityCreate) { |
87 | const view = activity.object as ViewObject | 87 | const view = activity.object as ViewObject |
88 | 88 | ||
89 | const { video } = await getOrCreateVideoAndAccountAndChannel(view.object) | 89 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: view.object }) |
90 | 90 | ||
91 | const actor = await ActorModel.loadByUrl(view.actor) | 91 | const actor = await ActorModel.loadByUrl(view.actor) |
92 | if (!actor) throw new Error('Unknown actor ' + view.actor) | 92 | if (!actor) throw new Error('Unknown actor ' + view.actor) |
@@ -103,7 +103,7 @@ async function processCreateView (byActor: ActorModel, activity: ActivityCreate) | |||
103 | async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) { | 103 | async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) { |
104 | const cacheFile = activity.object as CacheFileObject | 104 | const cacheFile = activity.object as CacheFileObject |
105 | 105 | ||
106 | const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFile.object) | 106 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFile.object }) |
107 | 107 | ||
108 | await createCacheFile(cacheFile, video, byActor) | 108 | await createCacheFile(cacheFile, video, byActor) |
109 | 109 | ||
@@ -120,7 +120,7 @@ async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateDat | |||
120 | const account = actor.Account | 120 | const account = actor.Account |
121 | if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url) | 121 | if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url) |
122 | 122 | ||
123 | const { video } = await getOrCreateVideoAndAccountAndChannel(videoAbuseToCreateData.object) | 123 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoAbuseToCreateData.object }) |
124 | 124 | ||
125 | return sequelizeTypescript.transaction(async t => { | 125 | return sequelizeTypescript.transaction(async t => { |
126 | const videoAbuseData = { | 126 | const videoAbuseData = { |
diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts index 9e1664fd8..631a9dde7 100644 --- a/server/lib/activitypub/process/process-like.ts +++ b/server/lib/activitypub/process/process-like.ts | |||
@@ -27,7 +27,7 @@ async function processLikeVideo (byActor: ActorModel, activity: ActivityLike) { | |||
27 | const byAccount = byActor.Account | 27 | const byAccount = byActor.Account |
28 | if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) | 28 | if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) |
29 | 29 | ||
30 | const { video } = await getOrCreateVideoAndAccountAndChannel(videoUrl) | 30 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoUrl }) |
31 | 31 | ||
32 | return sequelizeTypescript.transaction(async t => { | 32 | return sequelizeTypescript.transaction(async t => { |
33 | const rate = { | 33 | const rate = { |
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index 0eb5fa392..b78de6697 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts | |||
@@ -54,7 +54,7 @@ export { | |||
54 | async function processUndoLike (actorUrl: string, activity: ActivityUndo) { | 54 | async function processUndoLike (actorUrl: string, activity: ActivityUndo) { |
55 | const likeActivity = activity.object as ActivityLike | 55 | const likeActivity = activity.object as ActivityLike |
56 | 56 | ||
57 | const { video } = await getOrCreateVideoAndAccountAndChannel(likeActivity.object) | 57 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: likeActivity.object }) |
58 | 58 | ||
59 | return sequelizeTypescript.transaction(async t => { | 59 | return sequelizeTypescript.transaction(async t => { |
60 | const byAccount = await AccountModel.loadByUrl(actorUrl, t) | 60 | const byAccount = await AccountModel.loadByUrl(actorUrl, t) |
@@ -78,7 +78,7 @@ async function processUndoLike (actorUrl: string, activity: ActivityUndo) { | |||
78 | async function processUndoDislike (actorUrl: string, activity: ActivityUndo) { | 78 | async function processUndoDislike (actorUrl: string, activity: ActivityUndo) { |
79 | const dislike = activity.object.object as DislikeObject | 79 | const dislike = activity.object.object as DislikeObject |
80 | 80 | ||
81 | const { video } = await getOrCreateVideoAndAccountAndChannel(dislike.object) | 81 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: dislike.object }) |
82 | 82 | ||
83 | return sequelizeTypescript.transaction(async t => { | 83 | return sequelizeTypescript.transaction(async t => { |
84 | const byAccount = await AccountModel.loadByUrl(actorUrl, t) | 84 | const byAccount = await AccountModel.loadByUrl(actorUrl, t) |
@@ -102,7 +102,7 @@ async function processUndoDislike (actorUrl: string, activity: ActivityUndo) { | |||
102 | async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) { | 102 | async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) { |
103 | const cacheFileObject = activity.object.object as CacheFileObject | 103 | const cacheFileObject = activity.object.object as CacheFileObject |
104 | 104 | ||
105 | const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.object) | 105 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.object }) |
106 | 106 | ||
107 | return sequelizeTypescript.transaction(async t => { | 107 | return sequelizeTypescript.transaction(async t => { |
108 | const byActor = await ActorModel.loadByUrl(actorUrl) | 108 | const byActor = await ActorModel.loadByUrl(actorUrl) |
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index d3af1a181..935da5a54 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts | |||
@@ -48,7 +48,7 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) | |||
48 | return undefined | 48 | return undefined |
49 | } | 49 | } |
50 | 50 | ||
51 | const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id) | 51 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: videoObject.id }) |
52 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) | 52 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) |
53 | 53 | ||
54 | return updateVideoFromAP(video, videoObject, actor.Account, channelActor.VideoChannel, activity.to) | 54 | return updateVideoFromAP(video, videoObject, actor.Account, channelActor.VideoChannel, activity.to) |
@@ -64,7 +64,7 @@ async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUp | |||
64 | 64 | ||
65 | const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) | 65 | const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) |
66 | if (!redundancyModel) { | 66 | if (!redundancyModel) { |
67 | const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.id) | 67 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: cacheFileObject.id }) |
68 | return createCacheFile(cacheFileObject, video, byActor) | 68 | return createCacheFile(cacheFileObject, video, byActor) |
69 | } | 69 | } |
70 | 70 | ||
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index ffbd3a64e..4ca8bf659 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts | |||
@@ -94,7 +94,7 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []) { | |||
94 | try { | 94 | try { |
95 | // Maybe it's a reply to a video? | 95 | // Maybe it's a reply to a video? |
96 | // If yes, it's done: we resolved all the thread | 96 | // If yes, it's done: we resolved all the thread |
97 | const { video } = await getOrCreateVideoAndAccountAndChannel(url) | 97 | const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: url }) |
98 | 98 | ||
99 | if (comments.length !== 0) { | 99 | if (comments.length !== 0) { |
100 | const firstReply = comments[ comments.length - 1 ] | 100 | const firstReply = comments[ comments.length - 1 ] |
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 5150c9975..5aabd3e0d 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts | |||
@@ -3,7 +3,7 @@ import * as sequelize from 'sequelize' | |||
3 | import * as magnetUtil from 'magnet-uri' | 3 | import * as magnetUtil from 'magnet-uri' |
4 | import { join } from 'path' | 4 | import { join } from 'path' |
5 | import * as request from 'request' | 5 | import * as request from 'request' |
6 | import { ActivityIconObject, ActivityVideoUrlObject, VideoState, ActivityUrlObject } from '../../../shared/index' | 6 | import { ActivityIconObject, ActivityUrlObject, ActivityVideoUrlObject, VideoState } from '../../../shared/index' |
7 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' | 7 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' |
8 | import { VideoPrivacy } from '../../../shared/models/videos' | 8 | import { VideoPrivacy } from '../../../shared/models/videos' |
9 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' | 9 | import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' |
@@ -28,6 +28,7 @@ import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub | |||
28 | import { createRates } from './video-rates' | 28 | import { createRates } from './video-rates' |
29 | import { addVideoShares, shareVideoByServerAndChannel } from './share' | 29 | import { addVideoShares, shareVideoByServerAndChannel } from './share' |
30 | import { AccountModel } from '../../models/account/account' | 30 | import { AccountModel } from '../../models/account/account' |
31 | import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video' | ||
31 | 32 | ||
32 | async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { | 33 | async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { |
33 | // If the video is not private and published, we federate it | 34 | // If the video is not private and published, we federate it |
@@ -50,13 +51,24 @@ async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, tr | |||
50 | } | 51 | } |
51 | } | 52 | } |
52 | 53 | ||
53 | function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) { | 54 | async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> { |
54 | const host = video.VideoChannel.Account.Actor.Server.host | 55 | const options = { |
56 | uri: videoUrl, | ||
57 | method: 'GET', | ||
58 | json: true, | ||
59 | activityPub: true | ||
60 | } | ||
55 | 61 | ||
56 | // We need to provide a callback, if no we could have an uncaught exception | 62 | logger.info('Fetching remote video %s.', videoUrl) |
57 | return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => { | 63 | |
58 | if (err) reject(err) | 64 | const { response, body } = await doRequest(options) |
59 | }) | 65 | |
66 | if (sanitizeAndCheckVideoTorrentObject(body) === false) { | ||
67 | logger.debug('Remote video JSON is not valid.', { body }) | ||
68 | return { response, videoObject: undefined } | ||
69 | } | ||
70 | |||
71 | return { response, videoObject: body } | ||
60 | } | 72 | } |
61 | 73 | ||
62 | async function fetchRemoteVideoDescription (video: VideoModel) { | 74 | async function fetchRemoteVideoDescription (video: VideoModel) { |
@@ -71,6 +83,15 @@ async function fetchRemoteVideoDescription (video: VideoModel) { | |||
71 | return body.description ? body.description : '' | 83 | return body.description ? body.description : '' |
72 | } | 84 | } |
73 | 85 | ||
86 | function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Function) { | ||
87 | const host = video.VideoChannel.Account.Actor.Server.host | ||
88 | |||
89 | // We need to provide a callback, if no we could have an uncaught exception | ||
90 | return request.get(REMOTE_SCHEME.HTTP + '://' + host + path, err => { | ||
91 | if (err) reject(err) | ||
92 | }) | ||
93 | } | ||
94 | |||
74 | function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) { | 95 | function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) { |
75 | const thumbnailName = video.getThumbnailName() | 96 | const thumbnailName = video.getThumbnailName() |
76 | const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) | 97 | const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) |
@@ -82,94 +103,6 @@ function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) | |||
82 | return doRequestAndSaveToFile(options, thumbnailPath) | 103 | return doRequestAndSaveToFile(options, thumbnailPath) |
83 | } | 104 | } |
84 | 105 | ||
85 | async function videoActivityObjectToDBAttributes ( | ||
86 | videoChannel: VideoChannelModel, | ||
87 | videoObject: VideoTorrentObject, | ||
88 | to: string[] = [] | ||
89 | ) { | ||
90 | const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED | ||
91 | const duration = videoObject.duration.replace(/[^\d]+/, '') | ||
92 | |||
93 | let language: string | undefined | ||
94 | if (videoObject.language) { | ||
95 | language = videoObject.language.identifier | ||
96 | } | ||
97 | |||
98 | let category: number | undefined | ||
99 | if (videoObject.category) { | ||
100 | category = parseInt(videoObject.category.identifier, 10) | ||
101 | } | ||
102 | |||
103 | let licence: number | undefined | ||
104 | if (videoObject.licence) { | ||
105 | licence = parseInt(videoObject.licence.identifier, 10) | ||
106 | } | ||
107 | |||
108 | const description = videoObject.content || null | ||
109 | const support = videoObject.support || null | ||
110 | |||
111 | return { | ||
112 | name: videoObject.name, | ||
113 | uuid: videoObject.uuid, | ||
114 | url: videoObject.id, | ||
115 | category, | ||
116 | licence, | ||
117 | language, | ||
118 | description, | ||
119 | support, | ||
120 | nsfw: videoObject.sensitive, | ||
121 | commentsEnabled: videoObject.commentsEnabled, | ||
122 | waitTranscoding: videoObject.waitTranscoding, | ||
123 | state: videoObject.state, | ||
124 | channelId: videoChannel.id, | ||
125 | duration: parseInt(duration, 10), | ||
126 | createdAt: new Date(videoObject.published), | ||
127 | publishedAt: new Date(videoObject.published), | ||
128 | // FIXME: updatedAt does not seems to be considered by Sequelize | ||
129 | updatedAt: new Date(videoObject.updated), | ||
130 | views: videoObject.views, | ||
131 | likes: 0, | ||
132 | dislikes: 0, | ||
133 | remote: true, | ||
134 | privacy | ||
135 | } | ||
136 | } | ||
137 | |||
138 | function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) { | ||
139 | const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] | ||
140 | |||
141 | if (fileUrls.length === 0) { | ||
142 | throw new Error('Cannot find video files for ' + videoCreated.url) | ||
143 | } | ||
144 | |||
145 | const attributes: VideoFileModel[] = [] | ||
146 | for (const fileUrl of fileUrls) { | ||
147 | // Fetch associated magnet uri | ||
148 | const magnet = videoObject.url.find(u => { | ||
149 | return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height | ||
150 | }) | ||
151 | |||
152 | if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) | ||
153 | |||
154 | const parsed = magnetUtil.decode(magnet.href) | ||
155 | if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) { | ||
156 | throw new Error('Cannot parse magnet URI ' + magnet.href) | ||
157 | } | ||
158 | |||
159 | const attribute = { | ||
160 | extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ], | ||
161 | infoHash: parsed.infoHash, | ||
162 | resolution: fileUrl.height, | ||
163 | size: fileUrl.size, | ||
164 | videoId: videoCreated.id, | ||
165 | fps: fileUrl.fps | ||
166 | } as VideoFileModel | ||
167 | attributes.push(attribute) | ||
168 | } | ||
169 | |||
170 | return attributes | ||
171 | } | ||
172 | |||
173 | function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { | 106 | function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) { |
174 | const channel = videoObject.attributedTo.find(a => a.type === 'Group') | 107 | const channel = videoObject.attributedTo.find(a => a.type === 'Group') |
175 | if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) | 108 | if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url) |
@@ -177,51 +110,6 @@ function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject | |||
177 | return getOrCreateActorAndServerAndModel(channel.id) | 110 | return getOrCreateActorAndServerAndModel(channel.id) |
178 | } | 111 | } |
179 | 112 | ||
180 | async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { | ||
181 | logger.debug('Adding remote video %s.', videoObject.id) | ||
182 | |||
183 | const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { | ||
184 | const sequelizeOptions = { transaction: t } | ||
185 | |||
186 | const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) | ||
187 | const video = VideoModel.build(videoData) | ||
188 | |||
189 | const videoCreated = await video.save(sequelizeOptions) | ||
190 | |||
191 | // Process files | ||
192 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject) | ||
193 | if (videoFileAttributes.length === 0) { | ||
194 | throw new Error('Cannot find valid files for video %s ' + videoObject.url) | ||
195 | } | ||
196 | |||
197 | const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) | ||
198 | await Promise.all(videoFilePromises) | ||
199 | |||
200 | // Process tags | ||
201 | const tags = videoObject.tag.map(t => t.name) | ||
202 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | ||
203 | await videoCreated.$set('Tags', tagInstances, sequelizeOptions) | ||
204 | |||
205 | // Process captions | ||
206 | const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { | ||
207 | return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t) | ||
208 | }) | ||
209 | await Promise.all(videoCaptionsPromises) | ||
210 | |||
211 | logger.info('Remote video with uuid %s inserted.', videoObject.uuid) | ||
212 | |||
213 | videoCreated.VideoChannel = channelActor.VideoChannel | ||
214 | return videoCreated | ||
215 | }) | ||
216 | |||
217 | const p = generateThumbnailFromUrl(videoCreated, videoObject.icon) | ||
218 | .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) | ||
219 | |||
220 | if (waitThumbnail === true) await p | ||
221 | |||
222 | return videoCreated | ||
223 | } | ||
224 | |||
225 | type SyncParam = { | 113 | type SyncParam = { |
226 | likes: boolean | 114 | likes: boolean |
227 | dislikes: boolean | 115 | dislikes: boolean |
@@ -230,28 +118,7 @@ type SyncParam = { | |||
230 | thumbnail: boolean | 118 | thumbnail: boolean |
231 | refreshVideo: boolean | 119 | refreshVideo: boolean |
232 | } | 120 | } |
233 | async function getOrCreateVideoAndAccountAndChannel ( | 121 | async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: VideoTorrentObject, syncParam: SyncParam) { |
234 | videoObject: VideoTorrentObject | string, | ||
235 | syncParam: SyncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } | ||
236 | ) { | ||
237 | const videoUrl = typeof videoObject === 'string' ? videoObject : videoObject.id | ||
238 | |||
239 | let videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoUrl) | ||
240 | if (videoFromDatabase) { | ||
241 | const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase) | ||
242 | if (syncParam.refreshVideo === true) videoFromDatabase = await p | ||
243 | |||
244 | return { video: videoFromDatabase } | ||
245 | } | ||
246 | |||
247 | const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) | ||
248 | if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) | ||
249 | |||
250 | const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo) | ||
251 | const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail) | ||
252 | |||
253 | // Process outside the transaction because we could fetch remote data | ||
254 | |||
255 | logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) | 122 | logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) |
256 | 123 | ||
257 | const jobPayloads: ActivitypubHttpFetcherPayload[] = [] | 124 | const jobPayloads: ActivitypubHttpFetcherPayload[] = [] |
@@ -285,54 +152,37 @@ async function getOrCreateVideoAndAccountAndChannel ( | |||
285 | } | 152 | } |
286 | 153 | ||
287 | await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })) | 154 | await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })) |
288 | |||
289 | return { video } | ||
290 | } | 155 | } |
291 | 156 | ||
292 | async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoTorrentObject }> { | 157 | async function getOrCreateVideoAndAccountAndChannel (options: { |
293 | const options = { | 158 | videoObject: VideoTorrentObject | string, |
294 | uri: videoUrl, | 159 | syncParam?: SyncParam, |
295 | method: 'GET', | 160 | fetchType?: VideoFetchByUrlType |
296 | json: true, | 161 | }) { |
297 | activityPub: true | 162 | // Default params |
298 | } | 163 | const syncParam = options.syncParam || { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true, refreshVideo: false } |
164 | const fetchType = options.fetchType || 'all' | ||
299 | 165 | ||
300 | logger.info('Fetching remote video %s.', videoUrl) | 166 | // Get video url |
167 | const videoUrl = typeof options.videoObject === 'string' ? options.videoObject : options.videoObject.id | ||
301 | 168 | ||
302 | const { response, body } = await doRequest(options) | 169 | let videoFromDatabase = await fetchVideoByUrl(videoUrl, fetchType) |
170 | if (videoFromDatabase) { | ||
171 | const p = retryTransactionWrapper(refreshVideoIfNeeded, videoFromDatabase, fetchType, syncParam) | ||
172 | if (syncParam.refreshVideo === true) videoFromDatabase = await p | ||
303 | 173 | ||
304 | if (sanitizeAndCheckVideoTorrentObject(body) === false) { | 174 | return { video: videoFromDatabase } |
305 | logger.debug('Remote video JSON is not valid.', { body }) | ||
306 | return { response, videoObject: undefined } | ||
307 | } | 175 | } |
308 | 176 | ||
309 | return { response, videoObject: body } | 177 | const { videoObject: fetchedVideo } = await fetchRemoteVideo(videoUrl) |
310 | } | 178 | if (!fetchedVideo) throw new Error('Cannot fetch remote video with url: ' + videoUrl) |
311 | |||
312 | async function refreshVideoIfNeeded (video: VideoModel): Promise<VideoModel> { | ||
313 | if (!video.isOutdated()) return video | ||
314 | |||
315 | try { | ||
316 | const { response, videoObject } = await fetchRemoteVideo(video.url) | ||
317 | if (response.statusCode === 404) { | ||
318 | // Video does not exist anymore | ||
319 | await video.destroy() | ||
320 | return undefined | ||
321 | } | ||
322 | 179 | ||
323 | if (videoObject === undefined) { | 180 | const channelActor = await getOrCreateVideoChannelFromVideoObject(fetchedVideo) |
324 | logger.warn('Cannot refresh remote video: invalid body.') | 181 | const video = await retryTransactionWrapper(createVideo, fetchedVideo, channelActor, syncParam.thumbnail) |
325 | return video | ||
326 | } | ||
327 | 182 | ||
328 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) | 183 | await syncVideoExternalAttributes(video, fetchedVideo, syncParam) |
329 | const account = await AccountModel.load(channelActor.VideoChannel.accountId) | ||
330 | 184 | ||
331 | return updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel) | 185 | return { video } |
332 | } catch (err) { | ||
333 | logger.warn('Cannot refresh video.', { err }) | ||
334 | return video | ||
335 | } | ||
336 | } | 186 | } |
337 | 187 | ||
338 | async function updateVideoFromAP ( | 188 | async function updateVideoFromAP ( |
@@ -433,12 +283,7 @@ export { | |||
433 | fetchRemoteVideoStaticFile, | 283 | fetchRemoteVideoStaticFile, |
434 | fetchRemoteVideoDescription, | 284 | fetchRemoteVideoDescription, |
435 | generateThumbnailFromUrl, | 285 | generateThumbnailFromUrl, |
436 | videoActivityObjectToDBAttributes, | 286 | getOrCreateVideoChannelFromVideoObject |
437 | videoFileActivityUrlToDBAttributes, | ||
438 | createVideo, | ||
439 | getOrCreateVideoChannelFromVideoObject, | ||
440 | addVideoShares, | ||
441 | createRates | ||
442 | } | 287 | } |
443 | 288 | ||
444 | // --------------------------------------------------------------------------- | 289 | // --------------------------------------------------------------------------- |
@@ -448,3 +293,166 @@ function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideo | |||
448 | 293 | ||
449 | return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/') | 294 | return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/') |
450 | } | 295 | } |
296 | |||
297 | async function createVideo (videoObject: VideoTorrentObject, channelActor: ActorModel, waitThumbnail = false) { | ||
298 | logger.debug('Adding remote video %s.', videoObject.id) | ||
299 | |||
300 | const videoCreated: VideoModel = await sequelizeTypescript.transaction(async t => { | ||
301 | const sequelizeOptions = { transaction: t } | ||
302 | |||
303 | const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to) | ||
304 | const video = VideoModel.build(videoData) | ||
305 | |||
306 | const videoCreated = await video.save(sequelizeOptions) | ||
307 | |||
308 | // Process files | ||
309 | const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject) | ||
310 | if (videoFileAttributes.length === 0) { | ||
311 | throw new Error('Cannot find valid files for video %s ' + videoObject.url) | ||
312 | } | ||
313 | |||
314 | const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t })) | ||
315 | await Promise.all(videoFilePromises) | ||
316 | |||
317 | // Process tags | ||
318 | const tags = videoObject.tag.map(t => t.name) | ||
319 | const tagInstances = await TagModel.findOrCreateTags(tags, t) | ||
320 | await videoCreated.$set('Tags', tagInstances, sequelizeOptions) | ||
321 | |||
322 | // Process captions | ||
323 | const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { | ||
324 | return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t) | ||
325 | }) | ||
326 | await Promise.all(videoCaptionsPromises) | ||
327 | |||
328 | logger.info('Remote video with uuid %s inserted.', videoObject.uuid) | ||
329 | |||
330 | videoCreated.VideoChannel = channelActor.VideoChannel | ||
331 | return videoCreated | ||
332 | }) | ||
333 | |||
334 | const p = generateThumbnailFromUrl(videoCreated, videoObject.icon) | ||
335 | .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, { err })) | ||
336 | |||
337 | if (waitThumbnail === true) await p | ||
338 | |||
339 | return videoCreated | ||
340 | } | ||
341 | |||
342 | async function refreshVideoIfNeeded (videoArg: VideoModel, fetchedType: VideoFetchByUrlType, syncParam: SyncParam): Promise<VideoModel> { | ||
343 | // We need more attributes if the argument video was fetched with not enough joints | ||
344 | const video = fetchedType === 'all' ? videoArg : await VideoModel.loadByUrlAndPopulateAccount(videoArg.url) | ||
345 | |||
346 | if (!video.isOutdated()) return video | ||
347 | |||
348 | try { | ||
349 | const { response, videoObject } = await fetchRemoteVideo(video.url) | ||
350 | if (response.statusCode === 404) { | ||
351 | // Video does not exist anymore | ||
352 | await video.destroy() | ||
353 | return undefined | ||
354 | } | ||
355 | |||
356 | if (videoObject === undefined) { | ||
357 | logger.warn('Cannot refresh remote video: invalid body.') | ||
358 | return video | ||
359 | } | ||
360 | |||
361 | const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) | ||
362 | const account = await AccountModel.load(channelActor.VideoChannel.accountId) | ||
363 | |||
364 | await updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel) | ||
365 | await syncVideoExternalAttributes(video, videoObject, syncParam) | ||
366 | } catch (err) { | ||
367 | logger.warn('Cannot refresh video.', { err }) | ||
368 | return video | ||
369 | } | ||
370 | } | ||
371 | |||
372 | async function videoActivityObjectToDBAttributes ( | ||
373 | videoChannel: VideoChannelModel, | ||
374 | videoObject: VideoTorrentObject, | ||
375 | to: string[] = [] | ||
376 | ) { | ||
377 | const privacy = to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 ? VideoPrivacy.PUBLIC : VideoPrivacy.UNLISTED | ||
378 | const duration = videoObject.duration.replace(/[^\d]+/, '') | ||
379 | |||
380 | let language: string | undefined | ||
381 | if (videoObject.language) { | ||
382 | language = videoObject.language.identifier | ||
383 | } | ||
384 | |||
385 | let category: number | undefined | ||
386 | if (videoObject.category) { | ||
387 | category = parseInt(videoObject.category.identifier, 10) | ||
388 | } | ||
389 | |||
390 | let licence: number | undefined | ||
391 | if (videoObject.licence) { | ||
392 | licence = parseInt(videoObject.licence.identifier, 10) | ||
393 | } | ||
394 | |||
395 | const description = videoObject.content || null | ||
396 | const support = videoObject.support || null | ||
397 | |||
398 | return { | ||
399 | name: videoObject.name, | ||
400 | uuid: videoObject.uuid, | ||
401 | url: videoObject.id, | ||
402 | category, | ||
403 | licence, | ||
404 | language, | ||
405 | description, | ||
406 | support, | ||
407 | nsfw: videoObject.sensitive, | ||
408 | commentsEnabled: videoObject.commentsEnabled, | ||
409 | waitTranscoding: videoObject.waitTranscoding, | ||
410 | state: videoObject.state, | ||
411 | channelId: videoChannel.id, | ||
412 | duration: parseInt(duration, 10), | ||
413 | createdAt: new Date(videoObject.published), | ||
414 | publishedAt: new Date(videoObject.published), | ||
415 | // FIXME: updatedAt does not seems to be considered by Sequelize | ||
416 | updatedAt: new Date(videoObject.updated), | ||
417 | views: videoObject.views, | ||
418 | likes: 0, | ||
419 | dislikes: 0, | ||
420 | remote: true, | ||
421 | privacy | ||
422 | } | ||
423 | } | ||
424 | |||
425 | function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) { | ||
426 | const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] | ||
427 | |||
428 | if (fileUrls.length === 0) { | ||
429 | throw new Error('Cannot find video files for ' + videoCreated.url) | ||
430 | } | ||
431 | |||
432 | const attributes: VideoFileModel[] = [] | ||
433 | for (const fileUrl of fileUrls) { | ||
434 | // Fetch associated magnet uri | ||
435 | const magnet = videoObject.url.find(u => { | ||
436 | return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.height === fileUrl.height | ||
437 | }) | ||
438 | |||
439 | if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) | ||
440 | |||
441 | const parsed = magnetUtil.decode(magnet.href) | ||
442 | if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) { | ||
443 | throw new Error('Cannot parse magnet URI ' + magnet.href) | ||
444 | } | ||
445 | |||
446 | const attribute = { | ||
447 | extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ], | ||
448 | infoHash: parsed.infoHash, | ||
449 | resolution: fileUrl.height, | ||
450 | size: fileUrl.size, | ||
451 | videoId: videoCreated.id, | ||
452 | fps: fileUrl.fps | ||
453 | } as VideoFileModel | ||
454 | attributes.push(attribute) | ||
455 | } | ||
456 | |||
457 | return attributes | ||
458 | } | ||