From a3b472a12ec6e57dbe2f650419f8064864686eab Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 10 Aug 2022 11:51:13 +0200 Subject: Add ability to list imports of a channel sync --- server/controllers/api/users/me.ts | 11 +- server/controllers/api/video-channel.ts | 9 +- server/initializers/constants.ts | 2 +- .../0735-video-channel-sync-import-foreign-key.ts | 32 ++++ .../lib/job-queue/handlers/video-channel-import.ts | 20 ++- .../video-channel-sync-latest-scheduler.ts | 4 - server/lib/sync-channel.ts | 7 + server/lib/video-import.ts | 3 +- server/middlewares/validators/shared/index.ts | 1 + .../validators/shared/video-channel-syncs.ts | 24 +++ .../validators/videos/video-channel-sync.ts | 16 +- .../validators/videos/video-channels.ts | 14 +- .../middlewares/validators/videos/video-imports.ts | 19 ++- server/models/video/video-import.ts | 79 ++++++++-- .../api/check-params/channel-import-videos.ts | 172 +++++++++++++++++++++ server/tests/api/check-params/index.ts | 1 + server/tests/api/check-params/video-channels.ts | 113 +------------- server/tests/api/check-params/video-imports.ts | 9 ++ server/tests/api/videos/channel-import-videos.ts | 72 ++++++++- server/tests/api/videos/video-channel-syncs.ts | 12 ++ server/tests/api/videos/video-imports.ts | 9 ++ 21 files changed, 467 insertions(+), 162 deletions(-) create mode 100644 server/initializers/migrations/0735-video-channel-sync-import-foreign-key.ts create mode 100644 server/middlewares/validators/shared/video-channel-syncs.ts create mode 100644 server/tests/api/check-params/channel-import-videos.ts (limited to 'server') diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts index 595abcf95..00f580ee9 100644 --- a/server/controllers/api/users/me.ts +++ b/server/controllers/api/users/me.ts @@ -25,7 +25,13 @@ import { usersUpdateMeValidator, usersVideoRatingValidator } from '../../../middlewares' -import { deleteMeValidator, usersVideosValidator, videoImportsSortValidator, videosSortValidator } from '../../../middlewares/validators' +import { + deleteMeValidator, + getMyVideoImportsValidator, + usersVideosValidator, + videoImportsSortValidator, + videosSortValidator +} from '../../../middlewares/validators' import { updateAvatarValidator } from '../../../middlewares/validators/actor-image' import { AccountModel } from '../../../models/account/account' import { AccountVideoRateModel } from '../../../models/account/account-video-rate' @@ -60,6 +66,7 @@ meRouter.get('/me/videos/imports', videoImportsSortValidator, setDefaultSort, setDefaultPagination, + getMyVideoImportsValidator, asyncMiddleware(getUserVideoImports) ) @@ -138,7 +145,7 @@ async function getUserVideoImports (req: express.Request, res: express.Response) const resultList = await VideoImportModel.listUserVideoImportsForApi({ userId: user.id, - ...pick(req.query, [ 'targetUrl', 'start', 'count', 'sort' ]) + ...pick(req.query, [ 'targetUrl', 'start', 'count', 'sort', 'search', 'videoChannelSyncId' ]) }) return res.json(getFormattedObjects(resultList.data, resultList.total)) diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts index 89c7181bd..94285a78d 100644 --- a/server/controllers/api/video-channel.ts +++ b/server/controllers/api/video-channel.ts @@ -6,7 +6,7 @@ import { ActorFollowModel } from '@server/models/actor/actor-follow' import { getServerActor } from '@server/models/application/application' import { guessAdditionalAttributesFromQuery } from '@server/models/video/formatter/video-format-utils' import { MChannelBannerAccountDefault } from '@server/types/models' -import { ActorImageType, HttpStatusCode, VideoChannelCreate, VideoChannelUpdate } from '@shared/models' +import { ActorImageType, HttpStatusCode, VideoChannelCreate, VideoChannelUpdate, VideosImportInChannelCreate } from '@shared/models' import { auditLoggerFactory, getAuditIdFromRes, VideoChannelAuditView } from '../../helpers/audit-logger' import { resetSequelizeInstance } from '../../helpers/database-utils' import { buildNSFWFilter, createReqFiles, getCountVideos, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils' @@ -166,7 +166,7 @@ videoChannelRouter.get('/:nameWithHost/followers', videoChannelRouter.post('/:nameWithHost/import-videos', authenticate, asyncMiddleware(videoChannelsNameWithHostValidator), - videoChannelImportVideosValidator, + asyncMiddleware(videoChannelImportVideosValidator), ensureIsLocalChannel, ensureCanManageChannel, asyncMiddleware(ensureChannelOwnerCanUpload), @@ -418,13 +418,14 @@ async function listVideoChannelFollowers (req: express.Request, res: express.Res } async function importVideosInChannel (req: express.Request, res: express.Response) { - const { externalChannelUrl } = req.body + const { externalChannelUrl } = req.body as VideosImportInChannelCreate await JobQueue.Instance.createJob({ type: 'video-channel-import', payload: { externalChannelUrl, - videoChannelId: res.locals.videoChannel.id + videoChannelId: res.locals.videoChannel.id, + partOfChannelSyncId: res.locals.videoChannelSync?.id } }) diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 697a64d42..c2289ef36 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -25,7 +25,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 730 +const LAST_MIGRATION_VERSION = 735 // --------------------------------------------------------------------------- diff --git a/server/initializers/migrations/0735-video-channel-sync-import-foreign-key.ts b/server/initializers/migrations/0735-video-channel-sync-import-foreign-key.ts new file mode 100644 index 000000000..ffe0b11ab --- /dev/null +++ b/server/initializers/migrations/0735-video-channel-sync-import-foreign-key.ts @@ -0,0 +1,32 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize + db: any +}): Promise { + await utils.queryInterface.addColumn('videoImport', 'videoChannelSyncId', { + type: Sequelize.INTEGER, + defaultValue: null, + allowNull: true, + references: { + model: 'videoChannelSync', + key: 'id' + }, + onUpdate: 'CASCADE', + onDelete: 'SET NULL' + }, { transaction: utils.transaction }) +} + +async function down (utils: { + queryInterface: Sequelize.QueryInterface + transaction: Sequelize.Transaction +}) { + await utils.queryInterface.dropTable('videoChannelSync', { transaction: utils.transaction }) +} + +export { + up, + down +} diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts index 9bdb2d269..9aaad659e 100644 --- a/server/lib/job-queue/handlers/video-channel-import.ts +++ b/server/lib/job-queue/handlers/video-channel-import.ts @@ -3,6 +3,8 @@ import { logger } from '@server/helpers/logger' import { CONFIG } from '@server/initializers/config' import { synchronizeChannel } from '@server/lib/sync-channel' import { VideoChannelModel } from '@server/models/video/video-channel' +import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' +import { MChannelSync } from '@server/types/models' import { VideoChannelImportPayload } from '@shared/models' export async function processVideoChannelImport (job: Job) { @@ -12,13 +14,20 @@ export async function processVideoChannelImport (job: Job) { // Channel import requires only http upload to be allowed if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) { - logger.error('Cannot import channel as the HTTP upload is disabled') - return + throw new Error('Cannot import channel as the HTTP upload is disabled') } if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) { - logger.error('Cannot import channel as the synchronization is disabled') - return + throw new Error('Cannot import channel as the synchronization is disabled') + } + + let channelSync: MChannelSync + if (payload.partOfChannelSyncId) { + channelSync = await VideoChannelSyncModel.loadWithChannel(payload.partOfChannelSyncId) + + if (!channelSync) { + throw new Error('Unlnown channel sync specified in videos channel import') + } } const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId) @@ -28,7 +37,8 @@ export async function processVideoChannelImport (job: Job) { await synchronizeChannel({ channel: videoChannel, - externalChannelUrl: payload.externalChannelUrl + externalChannelUrl: payload.externalChannelUrl, + channelSync }) } catch (err) { logger.error(`Failed to import channel ${videoChannel.name}`, { err }) diff --git a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts index fd9a35299..491ddaa87 100644 --- a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts +++ b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts @@ -36,10 +36,6 @@ export class VideoChannelSyncLatestScheduler extends AbstractScheduler { const onlyAfter = sync.lastSyncAt || sync.createdAt - sync.state = VideoChannelSyncState.PROCESSING - sync.lastSyncAt = new Date() - await sync.save() - await synchronizeChannel({ channel, externalChannelUrl: sync.externalChannelUrl, diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts index 50f80e6f9..eb5ca1703 100644 --- a/server/lib/sync-channel.ts +++ b/server/lib/sync-channel.ts @@ -18,6 +18,12 @@ export async function synchronizeChannel (options: { }) { const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options + if (channelSync) { + channelSync.state = VideoChannelSyncState.PROCESSING + channelSync.lastSyncAt = new Date() + await channelSync.save() + } + const user = await UserModel.loadByChannelActorId(channel.actorId) const youtubeDL = new YoutubeDLWrapper( externalChannelUrl, @@ -70,6 +76,7 @@ export async function synchronizeChannel (options: { children.push(job) } + // Will update the channel sync status const parent: CreateJobArgument = { type: 'after-video-channel-import', payload: { diff --git a/server/lib/video-import.ts b/server/lib/video-import.ts index fb9306967..de95116aa 100644 --- a/server/lib/video-import.ts +++ b/server/lib/video-import.ts @@ -206,7 +206,8 @@ async function buildYoutubeDLImport (options: { videoImportAttributes: { targetUrl, state: VideoImportState.PENDING, - userId: user.id + userId: user.id, + videoChannelSyncId: channelSync?.id } }) diff --git a/server/middlewares/validators/shared/index.ts b/server/middlewares/validators/shared/index.ts index fa89d05f2..bbd03b248 100644 --- a/server/middlewares/validators/shared/index.ts +++ b/server/middlewares/validators/shared/index.ts @@ -4,6 +4,7 @@ export * from './utils' export * from './video-blacklists' export * from './video-captions' export * from './video-channels' +export * from './video-channel-syncs' export * from './video-comments' export * from './video-imports' export * from './video-ownerships' diff --git a/server/middlewares/validators/shared/video-channel-syncs.ts b/server/middlewares/validators/shared/video-channel-syncs.ts new file mode 100644 index 000000000..a6e51eb97 --- /dev/null +++ b/server/middlewares/validators/shared/video-channel-syncs.ts @@ -0,0 +1,24 @@ +import express from 'express' +import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' +import { HttpStatusCode } from '@shared/models' + +async function doesVideoChannelSyncIdExist (id: number, res: express.Response) { + const sync = await VideoChannelSyncModel.loadWithChannel(+id) + + if (!sync) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video channel sync not found' + }) + return false + } + + res.locals.videoChannelSync = sync + return true +} + +// --------------------------------------------------------------------------- + +export { + doesVideoChannelSyncIdExist +} diff --git a/server/middlewares/validators/videos/video-channel-sync.ts b/server/middlewares/validators/videos/video-channel-sync.ts index b18498243..081f09bba 100644 --- a/server/middlewares/validators/videos/video-channel-sync.ts +++ b/server/middlewares/validators/videos/video-channel-sync.ts @@ -3,10 +3,10 @@ import { body, param } from 'express-validator' import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc' import { logger } from '@server/helpers/logger' import { CONFIG } from '@server/initializers/config' -import { VideoChannelModel } from '@server/models/video/video-channel' import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models' import { areValidationErrors, doesVideoChannelIdExist } from '../shared' +import { doesVideoChannelSyncIdExist } from '../shared/video-channel-syncs' export const ensureSyncIsEnabled = (req: express.Request, res: express.Response, next: express.NextFunction) => { if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) { @@ -48,18 +48,8 @@ export const ensureSyncExists = [ async (req: express.Request, res: express.Response, next: express.NextFunction) => { if (areValidationErrors(req, res)) return - const syncId = parseInt(req.params.id, 10) - const sync = await VideoChannelSyncModel.loadWithChannel(syncId) - - if (!sync) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Synchronization not found' - }) - } - - res.locals.videoChannelSync = sync - res.locals.videoChannel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId) + if (!await doesVideoChannelSyncIdExist(+req.params.id, res)) return + if (!await doesVideoChannelIdExist(res.locals.videoChannelSync.videoChannelId, res)) return return next() } diff --git a/server/middlewares/validators/videos/video-channels.ts b/server/middlewares/validators/videos/video-channels.ts index 88f8b814d..d53c777fa 100644 --- a/server/middlewares/validators/videos/video-channels.ts +++ b/server/middlewares/validators/videos/video-channels.ts @@ -3,8 +3,9 @@ import { body, param, query } from 'express-validator' import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc' import { CONFIG } from '@server/initializers/config' import { MChannelAccountDefault } from '@server/types/models' +import { VideosImportInChannelCreate } from '@shared/models' import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { isBooleanValid, toBooleanOrNull } from '../../../helpers/custom-validators/misc' +import { isBooleanValid, isIdValid, toBooleanOrNull } from '../../../helpers/custom-validators/misc' import { isVideoChannelDescriptionValid, isVideoChannelDisplayNameValid, @@ -15,6 +16,7 @@ import { logger } from '../../../helpers/logger' import { ActorModel } from '../../../models/actor/actor' import { VideoChannelModel } from '../../../models/video/video-channel' import { areValidationErrors, checkUserQuota, doesVideoChannelNameWithHostExist } from '../shared' +import { doesVideoChannelSyncIdExist } from '../shared/video-channel-syncs' export const videoChannelsAddValidator = [ body('name').custom(isVideoChannelUsernameValid).withMessage('Should have a valid channel name'), @@ -145,11 +147,17 @@ export const videoChannelsListValidator = [ export const videoChannelImportVideosValidator = [ body('externalChannelUrl').custom(isUrlValid).withMessage('Should have a valid channel url'), - (req: express.Request, res: express.Response, next: express.NextFunction) => { + body('videoChannelSyncId') + .optional() + .custom(isIdValid).withMessage('Should have a valid channel sync id'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { logger.debug('Checking videoChannelImport parameters', { parameters: req.body }) if (areValidationErrors(req, res)) return + const body: VideosImportInChannelCreate = req.body + if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) { return res.fail({ status: HttpStatusCode.FORBIDDEN_403, @@ -157,6 +165,8 @@ export const videoChannelImportVideosValidator = [ }) } + if (body.videoChannelSyncId && !await doesVideoChannelSyncIdExist(body.videoChannelSyncId, res)) return + return next() } ] diff --git a/server/middlewares/validators/videos/video-imports.ts b/server/middlewares/validators/videos/video-imports.ts index 9c6d213c4..3115acb21 100644 --- a/server/middlewares/validators/videos/video-imports.ts +++ b/server/middlewares/validators/videos/video-imports.ts @@ -1,5 +1,5 @@ import express from 'express' -import { body, param } from 'express-validator' +import { body, param, query } from 'express-validator' import { isResolvingToUnicastOnly } from '@server/helpers/dns' import { isPreImportVideoAccepted } from '@server/lib/moderation' import { Hooks } from '@server/lib/plugins/hooks' @@ -92,6 +92,20 @@ const videoImportAddValidator = getCommonVideoEditAttributes().concat([ } ]) +const getMyVideoImportsValidator = [ + query('videoChannelSyncId') + .optional() + .custom(isIdValid).withMessage('Should have correct videoChannelSync id'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking getMyVideoImportsValidator parameters', { parameters: req.params }) + + if (areValidationErrors(req, res)) return + + return next() + } +] + const videoImportDeleteValidator = [ param('id') .custom(isIdValid).withMessage('Should have correct import id'), @@ -143,7 +157,8 @@ const videoImportCancelValidator = [ export { videoImportAddValidator, videoImportCancelValidator, - videoImportDeleteValidator + videoImportDeleteValidator, + getMyVideoImportsValidator } // --------------------------------------------------------------------------- diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts index b8e941623..da6b92c7a 100644 --- a/server/models/video/video-import.ts +++ b/server/models/video/video-import.ts @@ -1,4 +1,4 @@ -import { Op, WhereOptions } from 'sequelize' +import { IncludeOptions, Op, WhereOptions } from 'sequelize' import { AfterUpdate, AllowNull, @@ -22,8 +22,17 @@ import { isVideoImportStateValid, isVideoImportTargetUrlValid } from '../../help import { isVideoMagnetUriValid } from '../../helpers/custom-validators/videos' import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers/constants' import { UserModel } from '../user/user' -import { getSort, throwIfNotValid } from '../utils' +import { getSort, searchAttribute, throwIfNotValid } from '../utils' import { ScopeNames as VideoModelScopeNames, VideoModel } from './video' +import { VideoChannelSyncModel } from './video-channel-sync' + +const defaultVideoScope = () => { + return VideoModel.scope([ + VideoModelScopeNames.WITH_ACCOUNT_DETAILS, + VideoModelScopeNames.WITH_TAGS, + VideoModelScopeNames.WITH_THUMBNAILS + ]) +} @DefaultScope(() => ({ include: [ @@ -32,11 +41,11 @@ import { ScopeNames as VideoModelScopeNames, VideoModel } from './video' required: true }, { - model: VideoModel.scope([ - VideoModelScopeNames.WITH_ACCOUNT_DETAILS, - VideoModelScopeNames.WITH_TAGS, - VideoModelScopeNames.WITH_THUMBNAILS - ]), + model: defaultVideoScope(), + required: false + }, + { + model: VideoChannelSyncModel.unscoped(), required: false } ] @@ -113,6 +122,18 @@ export class VideoImportModel extends Model VideoChannelSyncModel) + @Column + videoChannelSyncId: number + + @BelongsTo(() => VideoChannelSyncModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'set null' + }) + VideoChannelSync: VideoChannelSyncModel + @AfterUpdate static deleteVideoIfFailed (instance: VideoImportModel, options) { if (instance.state === VideoImportState.FAILED) { @@ -132,23 +153,44 @@ export class VideoImportModel extends Model t.name) }) : undefined + const videoChannelSync = this.VideoChannelSync + ? { id: this.VideoChannelSync.id, externalChannelUrl: this.VideoChannelSync.externalChannelUrl } + : undefined + return { id: this.id, @@ -210,7 +256,8 @@ export class VideoImportModel extends Model !!i.videoChannelSync) + expect(importsWithSyncId).to.have.lengthOf(2) + + for (const videoImport of importsWithSyncId) { + expect(videoImport.videoChannelSync).to.exist + expect(videoImport.videoChannelSync.id).to.equal(server.store.videoChannelSync.id) + } + }) + + it('Should be able to filter imports by this sync id', async function () { + const { total, data } = await server.imports.getMyVideoImports({ videoChannelSyncId: server.store.videoChannelSync.id }) + + expect(total).to.equal(2) + expect(data).to.have.lengthOf(2) + + for (const videoImport of data) { + expect(videoImport.videoChannelSync).to.exist + expect(videoImport.videoChannelSync.id).to.equal(server.store.videoChannelSync.id) + } + }) + after(async function () { await server?.kill() }) diff --git a/server/tests/api/videos/video-channel-syncs.ts b/server/tests/api/videos/video-channel-syncs.ts index 229c01f68..835d3cb09 100644 --- a/server/tests/api/videos/video-channel-syncs.ts +++ b/server/tests/api/videos/video-channel-syncs.ts @@ -23,7 +23,10 @@ describe('Test channel synchronizations', function () { describe('Sync using ' + mode, function () { let server: PeerTubeServer let command: ChannelSyncsCommand + let startTestDate: Date + + let rootChannelSyncId: number const userInfo = { accessToken: '', username: 'user1', @@ -90,6 +93,7 @@ describe('Test channel synchronizations', function () { token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + rootChannelSyncId = videoChannelSync.id // Ensure any missing video not already fetched will be considered as new await changeDateForSync(videoChannelSync.id, '1970-01-01') @@ -208,6 +212,14 @@ describe('Test channel synchronizations', function () { } }) + it('Should list imports of a channel synchronization', async function () { + const { total, data } = await server.imports.getMyVideoImports({ videoChannelSyncId: rootChannelSyncId }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + expect(data[0].video.name).to.equal('test') + }) + it('Should remove user\'s channel synchronizations', async function () { await command.delete({ channelSyncId: userInfo.syncId }) diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts index a487062a2..f082d4bd7 100644 --- a/server/tests/api/videos/video-imports.ts +++ b/server/tests/api/videos/video-imports.ts @@ -228,6 +228,15 @@ describe('Test video imports', function () { expect(videoImports[0].targetUrl).to.equal(FIXTURE_URLS.youtube) }) + it('Should search in my imports', async function () { + const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ search: 'peertube2' }) + expect(total).to.equal(1) + expect(videoImports).to.have.lengthOf(1) + + expect(videoImports[0].magnetUri).to.equal(FIXTURE_URLS.magnet) + expect(videoImports[0].video.name).to.equal('super peertube2 video') + }) + it('Should have the video listed on the two instances', async function () { this.timeout(120_000) -- cgit v1.2.3