From fe3a55b071c99b346e9e9ab786f5d219e5a064cd Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 8 Jun 2018 20:34:37 +0200 Subject: [PATCH] Add video comments RSS --- scripts/travis.sh | 3 +- server/controllers/feeds.ts | 44 ++++++++- server/middlewares/validators/feeds.ts | 23 ++++- server/models/video/video-comment.ts | 22 +++++ server/tests/api/feeds/instance-feed.ts | 91 ------------------ server/tests/api/index-slow.ts | 1 - server/tests/feeds/feeds.ts | 120 ++++++++++++++++++++++++ server/tests/utils/feeds/feeds.ts | 10 +- 8 files changed, 211 insertions(+), 103 deletions(-) delete mode 100644 server/tests/api/feeds/instance-feed.ts create mode 100644 server/tests/feeds/feeds.ts diff --git a/scripts/travis.sh b/scripts/travis.sh index 79be23493..a5f604bb1 100755 --- a/scripts/travis.sh +++ b/scripts/travis.sh @@ -9,7 +9,8 @@ fi if [ "$1" = "misc" ]; then npm run build - mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/client.ts server/tests/activitypub.ts + mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/client.ts server/tests/activitypub.ts \ + server/tests/feeds/feeds.ts elif [ "$1" = "api" ]; then npm run build:server mocha --timeout 5000 --exit --require ts-node/register/type-check --bail server/tests/api/index.ts diff --git a/server/controllers/feeds.ts b/server/controllers/feeds.ts index 3a2b5ecca..c928dfacb 100644 --- a/server/controllers/feeds.ts +++ b/server/controllers/feeds.ts @@ -1,20 +1,27 @@ import * as express from 'express' import { CONFIG, FEEDS, ROUTE_CACHE_LIFETIME } from '../initializers/constants' -import { asyncMiddleware, feedsValidator, setDefaultSort, videosSortValidator } from '../middlewares' +import { asyncMiddleware, videoFeedsValidator, setDefaultSort, videosSortValidator, videoCommentsFeedsValidator } from '../middlewares' import { VideoModel } from '../models/video/video' import * as Feed from 'pfeed' import { AccountModel } from '../models/account/account' import { cacheRoute } from '../middlewares/cache' import { VideoChannelModel } from '../models/video/video-channel' +import { VideoCommentModel } from '../models/video/video-comment' const feedsRouter = express.Router() +feedsRouter.get('/feeds/video-comments.:format', + asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.FEEDS)), + asyncMiddleware(videoCommentsFeedsValidator), + asyncMiddleware(generateVideoCommentsFeed) +) + feedsRouter.get('/feeds/videos.:format', videosSortValidator, setDefaultSort, asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.FEEDS)), - asyncMiddleware(feedsValidator), - asyncMiddleware(generateFeed) + asyncMiddleware(videoFeedsValidator), + asyncMiddleware(generateVideoFeed) ) // --------------------------------------------------------------------------- @@ -25,7 +32,36 @@ export { // --------------------------------------------------------------------------- -async function generateFeed (req: express.Request, res: express.Response, next: express.NextFunction) { +async function generateVideoCommentsFeed (req: express.Request, res: express.Response, next: express.NextFunction) { + let feed = initFeed() + const start = 0 + + const videoId: number = res.locals.video ? res.locals.video.id : undefined + + const comments = await VideoCommentModel.listForFeed(start, FEEDS.COUNT, videoId) + + // Adding video items to the feed, one at a time + comments.forEach(comment => { + feed.addItem({ + title: `${comment.Video.name} - ${comment.Account.getDisplayName()}`, + id: comment.url, + link: comment.url, + content: comment.text, + author: [ + { + name: comment.Account.getDisplayName(), + link: comment.Account.Actor.url + } + ], + date: comment.createdAt + }) + }) + + // Now the feed generation is done, let's send it! + return sendFeed(feed, req, res) +} + +async function generateVideoFeed (req: express.Request, res: express.Response, next: express.NextFunction) { let feed = initFeed() const start = 0 diff --git a/server/middlewares/validators/feeds.ts b/server/middlewares/validators/feeds.ts index b55190559..3c8532bd9 100644 --- a/server/middlewares/validators/feeds.ts +++ b/server/middlewares/validators/feeds.ts @@ -7,12 +7,14 @@ import { logger } from '../../helpers/logger' import { areValidationErrors } from './utils' import { isValidRSSFeed } from '../../helpers/custom-validators/feeds' import { isVideoChannelExist } from '../../helpers/custom-validators/video-channels' +import { isVideoExist } from '../../helpers/custom-validators/videos' -const feedsValidator = [ +const videoFeedsValidator = [ param('format').optional().custom(isValidRSSFeed).withMessage('Should have a valid format (rss, atom, json)'), query('format').optional().custom(isValidRSSFeed).withMessage('Should have a valid format (rss, atom, json)'), query('accountId').optional().custom(isIdOrUUIDValid), query('accountName').optional().custom(isAccountNameValid), + query('videoChannelId').optional().custom(isIdOrUUIDValid), async (req: express.Request, res: express.Response, next: express.NextFunction) => { logger.debug('Checking feeds parameters', { parameters: req.query }) @@ -26,8 +28,25 @@ const feedsValidator = [ } ] +const videoCommentsFeedsValidator = [ + param('format').optional().custom(isValidRSSFeed).withMessage('Should have a valid format (rss, atom, json)'), + query('format').optional().custom(isValidRSSFeed).withMessage('Should have a valid format (rss, atom, json)'), + query('videoId').optional().custom(isIdOrUUIDValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking feeds parameters', { parameters: req.query }) + + if (areValidationErrors(req, res)) return + + if (req.query.videoId && !await isVideoExist(req.query.videoId, res)) return + + return next() + } +] + // --------------------------------------------------------------------------- export { - feedsValidator + videoFeedsValidator, + videoCommentsFeedsValidator } diff --git a/server/models/video/video-comment.ts b/server/models/video/video-comment.ts index 18398905e..353fb1a0e 100644 --- a/server/models/video/video-comment.ts +++ b/server/models/video/video-comment.ts @@ -340,6 +340,28 @@ export class VideoCommentModel extends Model { return VideoCommentModel.findAndCountAll(query) } + static listForFeed (start: number, count: number, videoId?: number) { + const query = { + order: [ [ 'createdAt', 'DESC' ] ], + start, + count, + where: {}, + include: [ + { + attributes: [ 'name' ], + model: VideoModel.unscoped(), + required: true + } + ] + } + + if (videoId) query.where['videoId'] = videoId + + return VideoCommentModel + .scope([ ScopeNames.WITH_ACCOUNT ]) + .findAll(query) + } + static async getStats () { const totalLocalVideoComments = await VideoCommentModel.count({ include: [ diff --git a/server/tests/api/feeds/instance-feed.ts b/server/tests/api/feeds/instance-feed.ts deleted file mode 100644 index e834e1db1..000000000 --- a/server/tests/api/feeds/instance-feed.ts +++ /dev/null @@ -1,91 +0,0 @@ -/* tslint:disable:no-unused-expression */ - -import * as chai from 'chai' -import 'mocha' -import { - getOEmbed, - getXMLfeed, - getJSONfeed, - flushTests, - killallServers, - ServerInfo, - setAccessTokensToServers, - uploadVideo, - flushAndRunMultipleServers, - wait -} from '../../utils' -import { runServer } from '../../utils/server/servers' -import { join } from 'path' -import * as libxmljs from 'libxmljs' - -chai.use(require('chai-xml')) -chai.use(require('chai-json-schema')) -chai.config.includeStack = true -const expect = chai.expect - -describe('Test instance-wide syndication feeds', () => { - let servers: ServerInfo[] = [] - - before(async function () { - this.timeout(30000) - - // Run servers - servers = await flushAndRunMultipleServers(2) - - await setAccessTokensToServers(servers) - - this.timeout(60000) - - const videoAttributes = { - name: 'my super name for server 1', - description: 'my super description for server 1', - fixture: 'video_short.webm' - } - await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes) - - await wait(10000) - }) - - it('should be well formed XML (covers RSS 2.0 and ATOM 1.0 endpoints)', async function () { - const rss = await getXMLfeed(servers[0].url) - expect(rss.text).xml.to.be.valid() - - const atom = await getXMLfeed(servers[0].url, 'atom') - expect(atom.text).xml.to.be.valid() - }) - - it('should be well formed JSON (covers JSON feed 1.0 endpoint)', async function () { - const json = await getJSONfeed(servers[0].url) - expect(JSON.parse(json.text)).to.be.jsonSchema({ 'type': 'object' }) - }) - - it('should contain a valid enclosure (covers RSS 2.0 endpoint)', async function () { - const rss = await getXMLfeed(servers[0].url) - const xmlDoc = libxmljs.parseXmlString(rss.text) - const xmlEnclosure = xmlDoc.get('/rss/channel/item/enclosure') - expect(xmlEnclosure).to.exist - expect(xmlEnclosure.attr('type').value()).to.be.equal('application/x-bittorrent') - expect(xmlEnclosure.attr('length').value()).to.be.equal('218910') - expect(xmlEnclosure.attr('url').value()).to.contain('720.torrent') - }) - - it('should contain a valid \'attachments\' object (covers JSON feed 1.0 endpoint)', async function () { - const json = await getJSONfeed(servers[0].url) - const jsonObj = JSON.parse(json.text) - expect(jsonObj.items.length).to.be.equal(1) - expect(jsonObj.items[0].attachments).to.exist - expect(jsonObj.items[0].attachments.length).to.be.eq(1) - expect(jsonObj.items[0].attachments[0].mime_type).to.be.eq('application/x-bittorrent') - expect(jsonObj.items[0].attachments[0].size_in_bytes).to.be.eq(218910) - expect(jsonObj.items[0].attachments[0].url).to.contain('720.torrent') - }) - - after(async function () { - killallServers(servers) - - // Keep the logs if the test failed - if (this['ok']) { - await flushTests() - } - }) -}) diff --git a/server/tests/api/index-slow.ts b/server/tests/api/index-slow.ts index 5f2f26095..cde546856 100644 --- a/server/tests/api/index-slow.ts +++ b/server/tests/api/index-slow.ts @@ -1,6 +1,5 @@ // Order of the tests we want to execute import './videos/video-transcoder' -import './feeds/instance-feed' import './videos/multiple-servers' import './server/follows' import './server/jobs' diff --git a/server/tests/feeds/feeds.ts b/server/tests/feeds/feeds.ts new file mode 100644 index 000000000..f65148f00 --- /dev/null +++ b/server/tests/feeds/feeds.ts @@ -0,0 +1,120 @@ +/* tslint:disable:no-unused-expression */ + +import * as chai from 'chai' +import 'mocha' +import { + doubleFollow, + flushAndRunMultipleServers, + flushTests, + getJSONfeed, + getXMLfeed, + killallServers, + ServerInfo, + setAccessTokensToServers, + uploadVideo, + wait +} from '../utils' +import { join } from 'path' +import * as libxmljs from 'libxmljs' +import { addVideoCommentThread } from '../utils/videos/video-comments' + +chai.use(require('chai-xml')) +chai.use(require('chai-json-schema')) +chai.config.includeStack = true +const expect = chai.expect + +describe('Test syndication feeds', () => { + let servers: ServerInfo[] = [] + + before(async function () { + this.timeout(120000) + + // Run servers + servers = await flushAndRunMultipleServers(2) + + await setAccessTokensToServers(servers) + await doubleFollow(servers[0], servers[1]) + + const videoAttributes = { + name: 'my super name for server 1', + description: 'my super description for server 1', + fixture: 'video_short.webm' + } + const res = await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes) + const videoId = res.body.video.id + + await addVideoCommentThread(servers[0].url, servers[0].accessToken, videoId, 'super comment 1') + await addVideoCommentThread(servers[0].url, servers[0].accessToken, videoId, 'super comment 2') + + await wait(10000) + }) + + describe('All feed', function () { + + it('Should be well formed XML (covers RSS 2.0 and ATOM 1.0 endpoints)', async function () { + for (const feed of [ 'video-comments' as 'video-comments', 'videos' as 'videos' ]) { + const rss = await getXMLfeed(servers[ 0 ].url, feed) + expect(rss.text).xml.to.be.valid() + + const atom = await getXMLfeed(servers[ 0 ].url, feed, 'atom') + expect(atom.text).xml.to.be.valid() + } + }) + + it('Should be well formed JSON (covers JSON feed 1.0 endpoint)', async function () { + for (const feed of [ 'video-comments' as 'video-comments', 'videos' as 'videos' ]) { + const json = await getJSONfeed(servers[ 0 ].url, feed) + expect(JSON.parse(json.text)).to.be.jsonSchema({ 'type': 'object' }) + } + }) + }) + + describe('Videos feed', function () { + it('Should contain a valid enclosure (covers RSS 2.0 endpoint)', async function () { + for (const server of servers) { + const rss = await getXMLfeed(server.url, 'videos') + const xmlDoc = libxmljs.parseXmlString(rss.text) + const xmlEnclosure = xmlDoc.get('/rss/channel/item/enclosure') + expect(xmlEnclosure).to.exist + expect(xmlEnclosure.attr('type').value()).to.be.equal('application/x-bittorrent') + expect(xmlEnclosure.attr('length').value()).to.be.equal('218910') + expect(xmlEnclosure.attr('url').value()).to.contain('720.torrent') + } + }) + + it('Should contain a valid \'attachments\' object (covers JSON feed 1.0 endpoint)', async function () { + for (const server of servers) { + const json = await getJSONfeed(server.url, 'videos') + const jsonObj = JSON.parse(json.text) + expect(jsonObj.items.length).to.be.equal(1) + expect(jsonObj.items[ 0 ].attachments).to.exist + expect(jsonObj.items[ 0 ].attachments.length).to.be.eq(1) + expect(jsonObj.items[ 0 ].attachments[ 0 ].mime_type).to.be.eq('application/x-bittorrent') + expect(jsonObj.items[ 0 ].attachments[ 0 ].size_in_bytes).to.be.eq(218910) + expect(jsonObj.items[ 0 ].attachments[ 0 ].url).to.contain('720.torrent') + } + }) + }) + + describe('Video comments feed', function () { + it('Should contain valid comments (covers JSON feed 1.0 endpoint)', async function () { + for (const server of servers) { + const json = await getJSONfeed(server.url, 'video-comments') + + const jsonObj = JSON.parse(json.text) + expect(jsonObj.items.length).to.be.equal(2) + expect(jsonObj.items[ 0 ].html_content).to.equal('super comment 2') + expect(jsonObj.items[ 1 ].html_content).to.equal('super comment 1') + } + }) + }) + + after(async function () { + killallServers(servers) + + // Keep the logs if the test failed + if (this['ok']) { + await flushTests() + } + }) +}) diff --git a/server/tests/utils/feeds/feeds.ts b/server/tests/utils/feeds/feeds.ts index 20e68cf3d..ffd23a1ad 100644 --- a/server/tests/utils/feeds/feeds.ts +++ b/server/tests/utils/feeds/feeds.ts @@ -1,8 +1,10 @@ import * as request from 'supertest' import { readFileBufferPromise } from '../../../helpers/core-utils' -function getXMLfeed (url: string, format?: string) { - const path = '/feeds/videos.xml' +type FeedType = 'videos' | 'video-comments' + +function getXMLfeed (url: string, feed: FeedType, format?: string) { + const path = '/feeds/' + feed + '.xml' return request(url) .get(path) @@ -12,8 +14,8 @@ function getXMLfeed (url: string, format?: string) { .expect('Content-Type', /xml/) } -function getJSONfeed (url: string) { - const path = '/feeds/videos.json' +function getJSONfeed (url: string, feed: FeedType) { + const path = '/feeds/' + feed + '.json' return request(url) .get(path) -- 2.41.0