From 38fa2065831b5f55be0d7f30f19a62c967397208 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 10 Nov 2017 14:48:08 +0100 Subject: Remove references to author --- server/lib/cache/videos-preview-cache.ts | 2 +- server/lib/friends.ts | 567 --------------------- .../video-file-optimizer-handler.ts | 4 +- .../video-file-transcoder-handler.ts | 4 +- server/lib/video-channel.ts | 2 +- 5 files changed, 6 insertions(+), 573 deletions(-) delete mode 100644 server/lib/friends.ts (limited to 'server/lib') diff --git a/server/lib/cache/videos-preview-cache.ts b/server/lib/cache/videos-preview-cache.ts index 0fe4d2f78..791ad1cbf 100644 --- a/server/lib/cache/videos-preview-cache.ts +++ b/server/lib/cache/videos-preview-cache.ts @@ -43,7 +43,7 @@ class VideosPreviewCache { } private async loadPreviews (key: string) { - const video = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(key) + const video = await db.Video.loadByUUIDAndPopulateAccountAndPodAndTags(key) if (!video) return undefined if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()) diff --git a/server/lib/friends.ts b/server/lib/friends.ts deleted file mode 100644 index 5c9baef47..000000000 --- a/server/lib/friends.ts +++ /dev/null @@ -1,567 +0,0 @@ -import * as request from 'request' -import * as Sequelize from 'sequelize' -import * as Bluebird from 'bluebird' -import { join } from 'path' - -import { database as db } from '../initializers/database' -import { - API_VERSION, - CONFIG, - REQUESTS_IN_PARALLEL, - REQUEST_ENDPOINTS, - REQUEST_ENDPOINT_ACTIONS, - REMOTE_SCHEME, - STATIC_PATHS -} from '../initializers' -import { - logger, - getMyPublicCert, - makeSecureRequest, - makeRetryRequest -} from '../helpers' -import { - RequestScheduler, - RequestSchedulerOptions, - - RequestVideoQaduScheduler, - RequestVideoQaduSchedulerOptions, - - RequestVideoEventScheduler, - RequestVideoEventSchedulerOptions -} from './request' -import { - PodInstance, - VideoInstance -} from '../models' -import { - RequestEndpoint, - RequestVideoEventType, - RequestVideoQaduType, - RemoteVideoCreateData, - RemoteVideoUpdateData, - RemoteVideoRemoveData, - RemoteVideoReportAbuseData, - ResultList, - RemoteVideoRequestType, - Pod as FormattedPod, - RemoteVideoChannelCreateData, - RemoteVideoChannelUpdateData, - RemoteVideoChannelRemoveData, - RemoteVideoAuthorCreateData, - RemoteVideoAuthorRemoveData -} from '../../shared' - -type QaduParam = { videoId: number, type: RequestVideoQaduType } -type EventParam = { videoId: number, type: RequestVideoEventType } - -const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] - -const requestScheduler = new RequestScheduler() -const requestVideoQaduScheduler = new RequestVideoQaduScheduler() -const requestVideoEventScheduler = new RequestVideoEventScheduler() - -function activateSchedulers () { - requestScheduler.activate() - requestVideoQaduScheduler.activate() - requestVideoEventScheduler.activate() -} - -function addVideoToFriends (videoData: RemoteVideoCreateData, transaction: Sequelize.Transaction) { - const options = { - type: ENDPOINT_ACTIONS.ADD_VIDEO, - endpoint: REQUEST_ENDPOINTS.VIDEOS, - data: videoData, - transaction - } - return createRequest(options) -} - -function updateVideoToFriends (videoData: RemoteVideoUpdateData, transaction: Sequelize.Transaction) { - const options = { - type: ENDPOINT_ACTIONS.UPDATE_VIDEO, - endpoint: REQUEST_ENDPOINTS.VIDEOS, - data: videoData, - transaction - } - return createRequest(options) -} - -function removeVideoToFriends (videoParams: RemoteVideoRemoveData, transaction?: Sequelize.Transaction) { - const options = { - type: ENDPOINT_ACTIONS.REMOVE_VIDEO, - endpoint: REQUEST_ENDPOINTS.VIDEOS, - data: videoParams, - transaction - } - return createRequest(options) -} - -function addVideoAuthorToFriends (authorData: RemoteVideoAuthorCreateData, transaction: Sequelize.Transaction) { - const options = { - type: ENDPOINT_ACTIONS.ADD_AUTHOR, - endpoint: REQUEST_ENDPOINTS.VIDEOS, - data: authorData, - transaction - } - return createRequest(options) -} - -function removeVideoAuthorToFriends (authorData: RemoteVideoAuthorRemoveData, transaction?: Sequelize.Transaction) { - const options = { - type: ENDPOINT_ACTIONS.REMOVE_AUTHOR, - endpoint: REQUEST_ENDPOINTS.VIDEOS, - data: authorData, - transaction - } - return createRequest(options) -} - -function addVideoChannelToFriends (videoChannelData: RemoteVideoChannelCreateData, transaction: Sequelize.Transaction) { - const options = { - type: ENDPOINT_ACTIONS.ADD_CHANNEL, - endpoint: REQUEST_ENDPOINTS.VIDEOS, - data: videoChannelData, - transaction - } - return createRequest(options) -} - -function updateVideoChannelToFriends (videoChannelData: RemoteVideoChannelUpdateData, transaction: Sequelize.Transaction) { - const options = { - type: ENDPOINT_ACTIONS.UPDATE_CHANNEL, - endpoint: REQUEST_ENDPOINTS.VIDEOS, - data: videoChannelData, - transaction - } - return createRequest(options) -} - -function removeVideoChannelToFriends (videoChannelParams: RemoteVideoChannelRemoveData, transaction?: Sequelize.Transaction) { - const options = { - type: ENDPOINT_ACTIONS.REMOVE_CHANNEL, - endpoint: REQUEST_ENDPOINTS.VIDEOS, - data: videoChannelParams, - transaction - } - return createRequest(options) -} - -function reportAbuseVideoToFriend (reportData: RemoteVideoReportAbuseData, video: VideoInstance, transaction: Sequelize.Transaction) { - const options = { - type: ENDPOINT_ACTIONS.REPORT_ABUSE, - endpoint: REQUEST_ENDPOINTS.VIDEOS, - data: reportData, - toIds: [ video.VideoChannel.Author.podId ], - transaction - } - return createRequest(options) -} - -function quickAndDirtyUpdateVideoToFriends (qaduParam: QaduParam, transaction?: Sequelize.Transaction) { - const options = { - videoId: qaduParam.videoId, - type: qaduParam.type, - transaction - } - return createVideoQaduRequest(options) -} - -function quickAndDirtyUpdatesVideoToFriends (qadusParams: QaduParam[], transaction: Sequelize.Transaction) { - const tasks = [] - - qadusParams.forEach(qaduParams => { - tasks.push(quickAndDirtyUpdateVideoToFriends(qaduParams, transaction)) - }) - - return Promise.all(tasks) -} - -function addEventToRemoteVideo (eventParam: EventParam, transaction?: Sequelize.Transaction) { - const options = { - videoId: eventParam.videoId, - type: eventParam.type, - transaction - } - return createVideoEventRequest(options) -} - -function addEventsToRemoteVideo (eventsParams: EventParam[], transaction: Sequelize.Transaction) { - const tasks = [] - - for (const eventParams of eventsParams) { - tasks.push(addEventToRemoteVideo(eventParams, transaction)) - } - - return Promise.all(tasks) -} - -async function hasFriends () { - const count = await db.Pod.countAll() - - return count !== 0 -} - -async function makeFriends (hosts: string[]) { - const podsScore = {} - - logger.info('Make friends!') - const cert = await getMyPublicCert() - - for (const host of hosts) { - await computeForeignPodsList(host, podsScore) - } - - logger.debug('Pods scores computed.', { podsScore: podsScore }) - - const podsList = computeWinningPods(hosts, podsScore) - logger.debug('Pods that we keep.', { podsToKeep: podsList }) - - return makeRequestsToWinningPods(cert, podsList) -} - -async function quitFriends () { - // Stop pool requests - requestScheduler.deactivate() - - try { - await requestScheduler.flush() - - await requestVideoQaduScheduler.flush() - - const pods = await db.Pod.list() - const requestParams = { - method: 'POST' as 'POST', - path: '/api/' + API_VERSION + '/remote/pods/remove', - toPod: null - } - - // Announce we quit them - // We don't care if the request fails - // The other pod will exclude us automatically after a while - try { - await Bluebird.map(pods, pod => { - requestParams.toPod = pod - - return makeSecureRequest(requestParams) - }, { concurrency: REQUESTS_IN_PARALLEL }) - } catch (err) { // Don't stop the process - logger.error('Some errors while quitting friends.', err) - } - - const tasks = [] - for (const pod of pods) { - tasks.push(pod.destroy()) - } - await Promise.all(pods) - - logger.info('Removed all remote videos.') - - requestScheduler.activate() - } catch (err) { - // Don't forget to re activate the scheduler, even if there was an error - requestScheduler.activate() - - throw err - } -} - -async function sendOwnedDataToPod (podId: number) { - // First send authors - await sendOwnedAuthorsToPod(podId) - await sendOwnedChannelsToPod(podId) - await sendOwnedVideosToPod(podId) -} - -async function sendOwnedChannelsToPod (podId: number) { - const videoChannels = await db.VideoChannel.listOwned() - - const tasks: Promise[] = [] - for (const videoChannel of videoChannels) { - const remoteVideoChannel = videoChannel.toAddRemoteJSON() - const options = { - type: 'add-channel' as 'add-channel', - endpoint: REQUEST_ENDPOINTS.VIDEOS, - data: remoteVideoChannel, - toIds: [ podId ], - transaction: null - } - - const p = createRequest(options) - tasks.push(p) - } - - await Promise.all(tasks) -} - -async function sendOwnedAuthorsToPod (podId: number) { - const authors = await db.Author.listOwned() - const tasks: Promise[] = [] - - for (const author of authors) { - const remoteAuthor = author.toAddRemoteJSON() - const options = { - type: 'add-author' as 'add-author', - endpoint: REQUEST_ENDPOINTS.VIDEOS, - data: remoteAuthor, - toIds: [ podId ], - transaction: null - } - - const p = createRequest(options) - tasks.push(p) - } - - await Promise.all(tasks) -} - -async function sendOwnedVideosToPod (podId: number) { - const videosList = await db.Video.listOwnedAndPopulateAuthorAndTags() - const tasks: Bluebird[] = [] - - for (const video of videosList) { - const promise = video.toAddRemoteJSON() - .then(remoteVideo => { - const options = { - type: 'add-video' as 'add-video', - endpoint: REQUEST_ENDPOINTS.VIDEOS, - data: remoteVideo, - toIds: [ podId ], - transaction: null - } - return createRequest(options) - }) - .catch(err => { - logger.error('Cannot convert video to remote.', err) - // Don't break the process - return undefined - }) - - tasks.push(promise) - } - - await Promise.all(tasks) -} - -function fetchRemotePreview (video: VideoInstance) { - const host = video.VideoChannel.Author.Pod.host - const path = join(STATIC_PATHS.PREVIEWS, video.getPreviewName()) - - return request.get(REMOTE_SCHEME.HTTP + '://' + host + path) -} - -function fetchRemoteDescription (video: VideoInstance) { - const host = video.VideoChannel.Author.Pod.host - const path = video.getDescriptionPath() - - const requestOptions = { - url: REMOTE_SCHEME.HTTP + '://' + host + path, - json: true - } - - return new Promise((res, rej) => { - request.get(requestOptions, (err, response, body) => { - if (err) return rej(err) - - return res(body.description ? body.description : '') - }) - }) -} - -async function removeFriend (pod: PodInstance) { - const requestParams = { - method: 'POST' as 'POST', - path: '/api/' + API_VERSION + '/remote/pods/remove', - toPod: pod - } - - try { - await makeSecureRequest(requestParams) - } catch (err) { - logger.warn('Cannot notify friends %s we are quitting him.', pod.host, err) - } - - try { - await pod.destroy() - - logger.info('Removed friend %s.', pod.host) - } catch (err) { - logger.error('Cannot destroy friend %s.', pod.host, err) - } -} - -function getRequestScheduler () { - return requestScheduler -} - -function getRequestVideoQaduScheduler () { - return requestVideoQaduScheduler -} - -function getRequestVideoEventScheduler () { - return requestVideoEventScheduler -} - -// --------------------------------------------------------------------------- - -export { - activateSchedulers, - addVideoToFriends, - removeVideoAuthorToFriends, - updateVideoToFriends, - addVideoAuthorToFriends, - reportAbuseVideoToFriend, - quickAndDirtyUpdateVideoToFriends, - quickAndDirtyUpdatesVideoToFriends, - addEventToRemoteVideo, - addEventsToRemoteVideo, - hasFriends, - makeFriends, - quitFriends, - removeFriend, - removeVideoToFriends, - sendOwnedDataToPod, - getRequestScheduler, - getRequestVideoQaduScheduler, - getRequestVideoEventScheduler, - fetchRemotePreview, - addVideoChannelToFriends, - fetchRemoteDescription, - updateVideoChannelToFriends, - removeVideoChannelToFriends -} - -// --------------------------------------------------------------------------- - -async function computeForeignPodsList (host: string, podsScore: { [ host: string ]: number }) { - const result = await getForeignPodsList(host) - const foreignPodsList: { host: string }[] = result.data - - // Let's give 1 point to the pod we ask the friends list - foreignPodsList.push({ host }) - - for (const foreignPod of foreignPodsList) { - const foreignPodHost = foreignPod.host - - if (podsScore[foreignPodHost]) podsScore[foreignPodHost]++ - else podsScore[foreignPodHost] = 1 - } - - return undefined -} - -function computeWinningPods (hosts: string[], podsScore: { [ host: string ]: number }) { - // Build the list of pods to add - // Only add a pod if it exists in more than a half base pods - const podsList = [] - const baseScore = hosts.length / 2 - - for (const podHost of Object.keys(podsScore)) { - // If the pod is not me and with a good score we add it - if (isMe(podHost) === false && podsScore[podHost] > baseScore) { - podsList.push({ host: podHost }) - } - } - - return podsList -} - -function getForeignPodsList (host: string) { - return new Promise< ResultList >((res, rej) => { - const path = '/api/' + API_VERSION + '/remote/pods/list' - - request.post(REMOTE_SCHEME.HTTP + '://' + host + path, (err, response, body) => { - if (err) return rej(err) - - try { - const json: ResultList = JSON.parse(body) - return res(json) - } catch (err) { - return rej(err) - } - }) - }) -} - -async function makeRequestsToWinningPods (cert: string, podsList: PodInstance[]) { - // Stop pool requests - requestScheduler.deactivate() - // Flush pool requests - requestScheduler.forceSend() - - try { - await Bluebird.map(podsList, async pod => { - const params = { - url: REMOTE_SCHEME.HTTP + '://' + pod.host + '/api/' + API_VERSION + '/remote/pods/add', - method: 'POST' as 'POST', - json: { - host: CONFIG.WEBSERVER.HOST, - email: CONFIG.ADMIN.EMAIL, - publicKey: cert - } - } - - const { response, body } = await makeRetryRequest(params) - const typedBody = body as { cert: string, email: string } - - if (response.statusCode === 200) { - const podObj = db.Pod.build({ host: pod.host, publicKey: typedBody.cert, email: typedBody.email }) - - let podCreated: PodInstance - try { - podCreated = await podObj.save() - } catch (err) { - logger.error('Cannot add friend %s pod.', pod.host, err) - } - - // Add our videos to the request scheduler - sendOwnedDataToPod(podCreated.id) - .catch(err => logger.warn('Cannot send owned data to pod %d.', podCreated.id, err)) - } else { - logger.error('Status not 200 for %s pod.', pod.host) - } - }, { concurrency: REQUESTS_IN_PARALLEL }) - - logger.debug('makeRequestsToWinningPods finished.') - - requestScheduler.activate() - } catch (err) { - // Final callback, we've ended all the requests - // Now we made new friends, we can re activate the pool of requests - requestScheduler.activate() - } -} - -// Wrapper that populate "toIds" argument with all our friends if it is not specified -type CreateRequestOptions = { - type: RemoteVideoRequestType - endpoint: RequestEndpoint - data: Object - toIds?: number[] - transaction: Sequelize.Transaction -} -async function createRequest (options: CreateRequestOptions) { - if (options.toIds !== undefined) { - await requestScheduler.createRequest(options as RequestSchedulerOptions) - return undefined - } - - // If the "toIds" pods is not specified, we send the request to all our friends - const podIds = await db.Pod.listAllIds(options.transaction) - - const newOptions = Object.assign(options, { toIds: podIds }) - await requestScheduler.createRequest(newOptions) - - return undefined -} - -function createVideoQaduRequest (options: RequestVideoQaduSchedulerOptions) { - return requestVideoQaduScheduler.createRequest(options) -} - -function createVideoEventRequest (options: RequestVideoEventSchedulerOptions) { - return requestVideoEventScheduler.createRequest(options) -} - -function isMe (host: string) { - return host === CONFIG.WEBSERVER.HOST -} diff --git a/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts b/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts index ccded4721..f019c28bc 100644 --- a/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts +++ b/server/lib/jobs/transcoding-job-scheduler/video-file-optimizer-handler.ts @@ -7,7 +7,7 @@ import { addVideoToFriends } from '../../friends' import { JobScheduler } from '../job-scheduler' async function process (data: { videoUUID: string }, jobId: number) { - const video = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(data.videoUUID) + const video = await db.Video.loadByUUIDAndPopulateAccountAndPodAndTags(data.videoUUID) // No video, maybe deleted? if (!video) { logger.info('Do not process job %d, video does not exist.', jobId, { videoUUID: video.uuid }) @@ -30,7 +30,7 @@ async function onSuccess (jobId: number, video: VideoInstance) { logger.info('Job %d is a success.', jobId) // Maybe the video changed in database, refresh it - const videoDatabase = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(video.uuid) + const videoDatabase = await db.Video.loadByUUIDAndPopulateAccountAndPodAndTags(video.uuid) // Video does not exist anymore if (!videoDatabase) return undefined diff --git a/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts b/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts index 853645510..397b95795 100644 --- a/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts +++ b/server/lib/jobs/transcoding-job-scheduler/video-file-transcoder-handler.ts @@ -5,7 +5,7 @@ import { VideoInstance } from '../../../models' import { VideoResolution } from '../../../../shared' async function process (data: { videoUUID: string, resolution: VideoResolution }, jobId: number) { - const video = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(data.videoUUID) + const video = await db.Video.loadByUUIDAndPopulateAccountAndPodAndTags(data.videoUUID) // No video, maybe deleted? if (!video) { logger.info('Do not process job %d, video does not exist.', jobId, { videoUUID: video.uuid }) @@ -28,7 +28,7 @@ async function onSuccess (jobId: number, video: VideoInstance) { logger.info('Job %d is a success.', jobId) // Maybe the video changed in database, refresh it - const videoDatabase = await db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(video.uuid) + const videoDatabase = await db.Video.loadByUUIDAndPopulateAccountAndPodAndTags(video.uuid) // Video does not exist anymore if (!videoDatabase) return undefined diff --git a/server/lib/video-channel.ts b/server/lib/video-channel.ts index a6dd4d061..f81383ce8 100644 --- a/server/lib/video-channel.ts +++ b/server/lib/video-channel.ts @@ -11,7 +11,7 @@ async function createVideoChannel (videoChannelInfo: VideoChannelCreate, account name: videoChannelInfo.name, description: videoChannelInfo.description, remote: false, - authorId: account.id + accountId: account.id } const videoChannel = db.VideoChannel.build(videoChannelData) -- cgit v1.2.3