From 7d14d4d2ca82cc43c93b45bb1f90af975cfbf67c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 31 May 2019 16:30:11 +0200 Subject: Server: Bulk update videos support field --- server/controllers/api/video-channel.ts | 26 +++++++++++--- server/lib/activitypub/process/process-create.ts | 1 + server/lib/video-channel.ts | 16 +++++++-- .../validators/videos/video-channels.ts | 16 +++++++-- server/models/video/video.ts | 23 ++++++++++++ server/tests/api/check-params/video-channels.ts | 12 +++++-- server/tests/api/videos/video-channels.ts | 42 ++++++++++++++++++---- shared/extra-utils/videos/video-channels.ts | 9 ++--- shared/extra-utils/videos/videos.ts | 1 + .../videos/channel/video-channel-update.model.ts | 4 ++- support/doc/api/openapi.yaml | 36 ++++++++++++++----- 11 files changed, 154 insertions(+), 32 deletions(-) diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts index c98a39be2..81a03a62b 100644 --- a/server/controllers/api/video-channel.ts +++ b/server/controllers/api/video-channel.ts @@ -19,7 +19,7 @@ import { VideoChannelModel } from '../../models/video/video-channel' import { videoChannelsNameWithHostValidator, videosSortValidator } from '../../middlewares/validators' import { sendUpdateActor } from '../../lib/activitypub/send' import { VideoChannelCreate, VideoChannelUpdate } from '../../../shared' -import { createVideoChannel } from '../../lib/video-channel' +import { createVideoChannel, federateAllVideosOfChannel } from '../../lib/video-channel' import { buildNSFWFilter, createReqFiles, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils' import { setAsyncActorKeys } from '../../lib/activitypub' import { AccountModel } from '../../models/account/account' @@ -160,6 +160,7 @@ async function updateVideoChannel (req: express.Request, res: express.Response) const videoChannelFieldsSave = videoChannelInstance.toJSON() const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannelInstance.toFormattedJSON()) const videoChannelInfoToUpdate = req.body as VideoChannelUpdate + let doBulkVideoUpdate = false try { await sequelizeTypescript.transaction(async t => { @@ -167,9 +168,18 @@ async function updateVideoChannel (req: express.Request, res: express.Response) transaction: t } - if (videoChannelInfoToUpdate.displayName !== undefined) videoChannelInstance.set('name', videoChannelInfoToUpdate.displayName) - if (videoChannelInfoToUpdate.description !== undefined) videoChannelInstance.set('description', videoChannelInfoToUpdate.description) - if (videoChannelInfoToUpdate.support !== undefined) videoChannelInstance.set('support', videoChannelInfoToUpdate.support) + if (videoChannelInfoToUpdate.displayName !== undefined) videoChannelInstance.name = videoChannelInfoToUpdate.displayName + if (videoChannelInfoToUpdate.description !== undefined) videoChannelInstance.description = videoChannelInfoToUpdate.description + + if (videoChannelInfoToUpdate.support !== undefined) { + const oldSupportField = videoChannelInstance.support + videoChannelInstance.support = videoChannelInfoToUpdate.support + + if (videoChannelInfoToUpdate.bulkVideosSupportUpdate === true && oldSupportField !== videoChannelInfoToUpdate.support) { + doBulkVideoUpdate = true + await VideoModel.bulkUpdateSupportField(videoChannelInstance, t) + } + } const videoChannelInstanceUpdated = await videoChannelInstance.save(sequelizeOptions) await sendUpdateActor(videoChannelInstanceUpdated, t) @@ -179,6 +189,7 @@ async function updateVideoChannel (req: express.Request, res: express.Response) new VideoChannelAuditView(videoChannelInstanceUpdated.toFormattedJSON()), oldVideoChannelAuditKeys ) + logger.info('Video channel %s updated.', videoChannelInstance.Actor.url) }) } catch (err) { @@ -192,7 +203,12 @@ async function updateVideoChannel (req: express.Request, res: express.Response) throw err } - return res.type('json').status(204).end() + res.type('json').status(204).end() + + // Don't process in a transaction, and after the response because it could be long + if (doBulkVideoUpdate) { + await federateAllVideosOfChannel(videoChannelInstance) + } } async function removeVideoChannel (req: express.Request, res: express.Response) { diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 3b6eb45c4..daf846513 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -87,6 +87,7 @@ async function processCreateVideoComment (activity: ActivityCreate, byActor: Act commentObject.inReplyTo, { err } ) + return } const { comment, created } = await addVideoComment(video, commentObject.id) diff --git a/server/lib/video-channel.ts b/server/lib/video-channel.ts index 0fe95ca09..ee0482c36 100644 --- a/server/lib/video-channel.ts +++ b/server/lib/video-channel.ts @@ -3,7 +3,8 @@ import * as uuidv4 from 'uuid/v4' import { VideoChannelCreate } from '../../shared/models' import { AccountModel } from '../models/account/account' import { VideoChannelModel } from '../models/video/video-channel' -import { buildActorInstance, getVideoChannelActivityPubUrl } from './activitypub' +import { buildActorInstance, federateVideoIfNeeded, getVideoChannelActivityPubUrl } from './activitypub' +import { VideoModel } from '../models/video/video' async function createVideoChannel (videoChannelInfo: VideoChannelCreate, account: AccountModel, t: Sequelize.Transaction) { const uuid = uuidv4() @@ -33,8 +34,19 @@ async function createVideoChannel (videoChannelInfo: VideoChannelCreate, account return videoChannelCreated } +async function federateAllVideosOfChannel (videoChannel: VideoChannelModel) { + const videoIds = await VideoModel.getAllIdsFromChannel(videoChannel) + + for (const videoId of videoIds) { + const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId) + + await federateVideoIfNeeded(video, false) + } +} + // --------------------------------------------------------------------------- export { - createVideoChannel + createVideoChannel, + federateAllVideosOfChannel } diff --git a/server/middlewares/validators/videos/video-channels.ts b/server/middlewares/validators/videos/video-channels.ts index 4b26f0bc4..f5a59cacb 100644 --- a/server/middlewares/validators/videos/video-channels.ts +++ b/server/middlewares/validators/videos/video-channels.ts @@ -14,6 +14,7 @@ import { VideoChannelModel } from '../../../models/video/video-channel' import { areValidationErrors } from '../utils' import { isActorPreferredUsernameValid } from '../../../helpers/custom-validators/activitypub/actor' import { ActorModel } from '../../../models/activitypub/actor' +import { isBooleanValid } from '../../../helpers/custom-validators/misc' const videoChannelsAddValidator = [ body('name').custom(isActorPreferredUsernameValid).withMessage('Should have a valid channel name'), @@ -40,9 +41,18 @@ const videoChannelsAddValidator = [ const videoChannelsUpdateValidator = [ param('nameWithHost').exists().withMessage('Should have an video channel name with host'), - body('displayName').optional().custom(isVideoChannelNameValid).withMessage('Should have a valid display name'), - body('description').optional().custom(isVideoChannelDescriptionValid).withMessage('Should have a valid description'), - body('support').optional().custom(isVideoChannelSupportValid).withMessage('Should have a valid support text'), + body('displayName') + .optional() + .custom(isVideoChannelNameValid).withMessage('Should have a valid display name'), + body('description') + .optional() + .custom(isVideoChannelDescriptionValid).withMessage('Should have a valid description'), + body('support') + .optional() + .custom(isVideoChannelSupportValid).withMessage('Should have a valid support text'), + body('bulkVideosSupportUpdate') + .optional() + .custom(isBooleanValid).withMessage('Should have a valid bulkVideosSupportUpdate boolean field'), async (req: express.Request, res: express.Response, next: express.NextFunction) => { logger.debug('Checking videoChannelsUpdate parameters', { parameters: req.body }) diff --git a/server/models/video/video.ts b/server/models/video/video.ts index c0a7892a4..eccf0a4fa 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -1515,6 +1515,29 @@ export class VideoModel extends Model { .then(results => results.length === 1) } + static bulkUpdateSupportField (videoChannel: VideoChannelModel, t: Transaction) { + const options = { + where: { + channelId: videoChannel.id + }, + transaction: t + } + + return VideoModel.update({ support: videoChannel.support }, options) + } + + static getAllIdsFromChannel (videoChannel: VideoChannelModel) { + const query = { + attributes: [ 'id' ], + where: { + channelId: videoChannel.id + } + } + + return VideoModel.findAll(query) + .then(videos => videos.map(v => v.id)) + } + // threshold corresponds to how many video the field should have to be returned static async getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) { const serverActor = await getServerActor() diff --git a/server/tests/api/check-params/video-channels.ts b/server/tests/api/check-params/video-channels.ts index ff04f6b03..de88298d1 100644 --- a/server/tests/api/check-params/video-channels.ts +++ b/server/tests/api/check-params/video-channels.ts @@ -24,6 +24,7 @@ import { checkBadStartPagination } from '../../../../shared/extra-utils/requests/check-api-params' import { join } from 'path' +import { VideoChannelUpdate } from '../../../../shared/models/videos' const expect = chai.expect @@ -169,9 +170,11 @@ describe('Test video channels API validator', function () { }) describe('When updating a video channel', function () { - const baseCorrectParams = { + const baseCorrectParams: VideoChannelUpdate = { displayName: 'hello', - description: 'super description' + description: 'super description', + support: 'toto', + bulkVideosSupportUpdate: false } let path: string @@ -214,6 +217,11 @@ describe('Test video channels API validator', function () { await makePutBodyRequest({ url: server.url, path, token: server.accessToken, fields }) }) + it('Should fail with a bad bulkVideosSupportUpdate field', async function () { + const fields = immutableAssign(baseCorrectParams, { bulkVideosSupportUpdate: 'super' }) + await makePutBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + it('Should succeed with the correct parameters', async function () { await makePutBodyRequest({ url: server.url, diff --git a/server/tests/api/videos/video-channels.ts b/server/tests/api/videos/video-channels.ts index 2d298dd3f..4f600cae8 100644 --- a/server/tests/api/videos/video-channels.ts +++ b/server/tests/api/videos/video-channels.ts @@ -2,12 +2,12 @@ import * as chai from 'chai' import 'mocha' -import { User, Video, VideoChannel } from '../../../../shared/index' +import { User, Video, VideoChannel, VideoDetails } from '../../../../shared/index' import { cleanupTests, createUser, doubleFollow, - flushAndRunMultipleServers, + flushAndRunMultipleServers, getVideo, getVideoChannelVideos, testImage, updateVideo, @@ -79,7 +79,8 @@ describe('Test video channels', function () { // The channel is 1 is propagated to servers 2 { - const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, { name: 'my video name', channelId: secondVideoChannelId }) + const videoAttributesArg = { name: 'my video name', channelId: secondVideoChannelId, support: 'video support field' } + const res = await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, videoAttributesArg) videoUUID = res.body.video.uuid } @@ -201,12 +202,12 @@ describe('Test video channels', function () { }) it('Should update video channel', async function () { - this.timeout(5000) + this.timeout(15000) const videoChannelAttributes = { displayName: 'video channel updated', description: 'video channel description updated', - support: 'video channel support text updated' + support: 'support updated' } await updateVideoChannel(servers[0].url, servers[0].accessToken, 'second_video_channel', videoChannelAttributes) @@ -224,7 +225,36 @@ describe('Test video channels', function () { expect(res.body.data[0].name).to.equal('second_video_channel') expect(res.body.data[0].displayName).to.equal('video channel updated') expect(res.body.data[0].description).to.equal('video channel description updated') - expect(res.body.data[0].support).to.equal('video channel support text updated') + expect(res.body.data[0].support).to.equal('support updated') + } + }) + + it('Should not have updated the video support field', async function () { + for (const server of servers) { + const res = await getVideo(server.url, videoUUID) + const video: VideoDetails = res.body + + expect(video.support).to.equal('video support field') + } + }) + + it('Should update the channel support field and update videos too', async function () { + this.timeout(35000) + + const videoChannelAttributes = { + support: 'video channel support text updated', + bulkVideosSupportUpdate: true + } + + await updateVideoChannel(servers[0].url, servers[0].accessToken, 'second_video_channel', videoChannelAttributes) + + await waitJobs(servers) + + for (const server of servers) { + const res = await getVideo(server.url, videoUUID) + const video: VideoDetails = res.body + + expect(video.support).to.equal(videoChannelAttributes.support) } }) diff --git a/shared/extra-utils/videos/video-channels.ts b/shared/extra-utils/videos/video-channels.ts index b4755b486..3e79cf15a 100644 --- a/shared/extra-utils/videos/video-channels.ts +++ b/shared/extra-utils/videos/video-channels.ts @@ -74,12 +74,13 @@ function updateVideoChannel ( attributes: VideoChannelUpdate, expectedStatus = 204 ) { - const body = {} + const body: any = {} const path = '/api/v1/video-channels/' + channelName - if (attributes.displayName) body['displayName'] = attributes.displayName - if (attributes.description) body['description'] = attributes.description - if (attributes.support) body['support'] = attributes.support + if (attributes.displayName) body.displayName = attributes.displayName + if (attributes.description) body.description = attributes.description + if (attributes.support) body.support = attributes.support + if (attributes.bulkVideosSupportUpdate) body.bulkVideosSupportUpdate = attributes.bulkVideosSupportUpdate return request(url) .put(path) diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts index a4ca43f26..debaaf9a7 100644 --- a/shared/extra-utils/videos/videos.ts +++ b/shared/extra-utils/videos/videos.ts @@ -355,6 +355,7 @@ async function uploadVideo (url: string, accessToken: string, videoAttributesArg .set('Accept', 'application/json') .set('Authorization', 'Bearer ' + accessToken) .field('name', attributes.name) + .field('support', attributes.support) .field('nsfw', JSON.stringify(attributes.nsfw)) .field('commentsEnabled', JSON.stringify(attributes.commentsEnabled)) .field('downloadEnabled', JSON.stringify(attributes.downloadEnabled)) diff --git a/shared/models/videos/channel/video-channel-update.model.ts b/shared/models/videos/channel/video-channel-update.model.ts index 3626ce8a9..8dde9188b 100644 --- a/shared/models/videos/channel/video-channel-update.model.ts +++ b/shared/models/videos/channel/video-channel-update.model.ts @@ -1,5 +1,7 @@ export interface VideoChannelUpdate { - displayName: string + displayName?: string description?: string support?: string + + bulkVideosSupportUpdate?: boolean } diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yaml index a8a064fd0..f3f565694 100644 --- a/support/doc/api/openapi.yaml +++ b/support/doc/api/openapi.yaml @@ -1322,7 +1322,10 @@ paths: '204': $ref: '#/paths/~1users~1me/put/responses/204' requestBody: - $ref: '#/components/requestBodies/VideoChannelInput' + content: + application/json: + schema: + $ref: '#/components/schemas/VideoChannelCreate' '/video-channels/{channelHandle}': get: summary: Get a video channel by its id @@ -1349,7 +1352,10 @@ paths: '204': $ref: '#/paths/~1users~1me/put/responses/204' requestBody: - $ref: '#/components/requestBodies/VideoChannelInput' + content: + application/json: + schema: + $ref: '#/components/schemas/VideoChannelUpdate' delete: summary: Delete a video channel by its id security: @@ -1775,12 +1781,6 @@ components: type: array items: type: string - requestBodies: - VideoChannelInput: - content: - application/json: - schema: - $ref: '#/components/schemas/VideoChannelInput' securitySchemes: OAuth2: description: > @@ -2294,10 +2294,28 @@ components: - username - password - email - VideoChannelInput: + VideoChannelCreate: properties: name: type: string + displayName: + type: string description: type: string + support: + type: string + required: + - name + - displayName + VideoChannelUpdate: + properties: + displayName: + type: string + description: + type: string + support: + type: string + bulkVideosSupportUpdate: + type: boolean + description: 'Update all videos support field of this channel' -- cgit v1.2.3