From 244e76a552ef05a5067134b1065d26dd89246d8c Mon Sep 17 00:00:00 2001 From: Rigel Kent Date: Tue, 17 Apr 2018 00:49:04 +0200 Subject: feature: initial syndication feeds support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provides rss 2.0, atom 1.0 and json 1.0 feeds for videos (instance and account-wide) on listings and video-watch views. * still lacks redis caching * still lacks lastBuildDate support * still lacks channel-wide support * still lacks semantic annotation (for licenses, NSFW warnings, etc.) * still lacks love ( ˘ ³˘) * RSS: has MRSS support for torrent lists! * RSS: includes the first torrent in an enclosure * JSON: lists all torrents in the 'attachments' object * ATOM: lacking torrent listing support Advances #23 Partial implementation for the accountId generation in the client, which will need a hotfix to add a way to get the proper account id. --- server/controllers/feeds.ts | 136 ++++++++++++++++++++++++ server/controllers/index.ts | 5 +- server/helpers/custom-validators/feeds.ts | 23 +++++ server/middlewares/validators/feeds.ts | 35 +++++++ server/middlewares/validators/index.ts | 1 + server/models/account/account.ts | 6 +- server/models/video/video.ts | 166 +++++++++++++++++------------- 7 files changed, 298 insertions(+), 74 deletions(-) create mode 100644 server/controllers/feeds.ts create mode 100644 server/helpers/custom-validators/feeds.ts create mode 100644 server/middlewares/validators/feeds.ts (limited to 'server') diff --git a/server/controllers/feeds.ts b/server/controllers/feeds.ts new file mode 100644 index 000000000..b9d4c5d50 --- /dev/null +++ b/server/controllers/feeds.ts @@ -0,0 +1,136 @@ +import * as express from 'express' +import { CONFIG } from '../initializers' +import { asyncMiddleware, feedsValidator } from '../middlewares' +import { VideoModel } from '../models/video/video' +import * as Feed from 'pfeed' +import { ResultList } from '../../shared/models' +import { AccountModel } from '../models/account/account' + +const feedsRouter = express.Router() + +feedsRouter.get('/feeds/videos.:format', + asyncMiddleware(feedsValidator), + asyncMiddleware(generateFeed) +) + +// --------------------------------------------------------------------------- + +export { + feedsRouter +} + +// --------------------------------------------------------------------------- + +async function generateFeed (req: express.Request, res: express.Response, next: express.NextFunction) { + let feed = initFeed() + let feedStart = 0 + let feedCount = 10 + let feedSort = '-createdAt' + + let resultList: ResultList + const account: AccountModel = res.locals.account + + if (account) { + resultList = await VideoModel.listUserVideosForApi( + account.id, + feedStart, + feedCount, + feedSort, + true + ) + } else { + resultList = await VideoModel.listForApi( + feedStart, + feedCount, + feedSort, + req.query.filter, + true + ) + } + + // Adding video items to the feed, one at a time + resultList.data.forEach(video => { + const formattedVideoFiles = video.getFormattedVideoFilesJSON() + const torrents = formattedVideoFiles.map(videoFile => ({ + title: video.name, + url: videoFile.torrentUrl, + size_in_bytes: videoFile.size + })) + + feed.addItem({ + title: video.name, + id: video.url, + link: video.url, + description: video.getTruncatedDescription(), + content: video.description, + author: [ + { + name: video.VideoChannel.Account.getDisplayName(), + link: video.VideoChannel.Account.Actor.url + } + ], + date: video.publishedAt, + language: video.language, + nsfw: video.nsfw, + torrent: torrents + }) + }) + + // Now the feed generation is done, let's send it! + return sendFeed(feed, req, res) +} + +function initFeed () { + const webserverUrl = CONFIG.WEBSERVER.URL + + return new Feed({ + title: CONFIG.INSTANCE.NAME, + description: CONFIG.INSTANCE.SHORT_DESCRIPTION, + // updated: TODO: somehowGetLatestUpdate, // optional, default = today + id: webserverUrl, + link: webserverUrl, + image: webserverUrl + '/client/assets/images/icons/icon-96x96.png', + favicon: webserverUrl + '/client/assets/images/favicon.png', + copyright: `All rights reserved, unless otherwise specified in the terms specified at ${webserverUrl}/about` + + ` and potential licenses granted by each content's rightholder.`, + generator: `Toraifōsu`, // ^.~ + feedLinks: { + json: `${webserverUrl}/feeds/videos.json`, + atom: `${webserverUrl}/feeds/videos.atom`, + rss: `${webserverUrl}/feeds/videos.xml` + }, + author: { + name: 'instance admin of ' + CONFIG.INSTANCE.NAME, + email: CONFIG.ADMIN.EMAIL, + link: `${webserverUrl}/about` + } + }) +} + +function sendFeed (feed, req: express.Request, res: express.Response) { + const format = req.params.format + + if (format === 'atom' || format === 'atom1') { + res.set('Content-Type', 'application/atom+xml') + return res.send(feed.atom1()).end() + } + + if (format === 'json' || format === 'json1') { + res.set('Content-Type', 'application/json') + return res.send(feed.json1()).end() + } + + if (format === 'rss' || format === 'rss2') { + res.set('Content-Type', 'application/rss+xml') + return res.send(feed.rss2()).end() + } + + // We're in the ambiguous '.xml' case and we look at the format query parameter + if (req.query.format === 'atom' || req.query.format === 'atom1') { + res.set('Content-Type', 'application/atom+xml') + return res.send(feed.atom1()).end() + } + + res.set('Content-Type', 'application/rss+xml') + return res.send(feed.rss2()).end() +} diff --git a/server/controllers/index.ts b/server/controllers/index.ts index 457d0a12e..ff7928312 100644 --- a/server/controllers/index.ts +++ b/server/controllers/index.ts @@ -1,6 +1,7 @@ export * from './activitypub' -export * from './static' +export * from './api' export * from './client' +export * from './feeds' export * from './services' -export * from './api' +export * from './static' export * from './webfinger' diff --git a/server/helpers/custom-validators/feeds.ts b/server/helpers/custom-validators/feeds.ts new file mode 100644 index 000000000..638e814f0 --- /dev/null +++ b/server/helpers/custom-validators/feeds.ts @@ -0,0 +1,23 @@ +import { exists } from './misc' + +function isValidRSSFeed (value: string) { + if (!exists(value)) return false + + const feedExtensions = [ + 'xml', + 'json', + 'json1', + 'rss', + 'rss2', + 'atom', + 'atom1' + ] + + return feedExtensions.indexOf(value) !== -1 +} + +// --------------------------------------------------------------------------- + +export { + isValidRSSFeed +} diff --git a/server/middlewares/validators/feeds.ts b/server/middlewares/validators/feeds.ts new file mode 100644 index 000000000..6a8cfce86 --- /dev/null +++ b/server/middlewares/validators/feeds.ts @@ -0,0 +1,35 @@ +import * as express from 'express' +import { param, query } from 'express-validator/check' +import { isAccountIdExist, isAccountNameValid, isLocalAccountNameExist } from '../../helpers/custom-validators/accounts' +import { join } from 'path' +import { isIdOrUUIDValid } from '../../helpers/custom-validators/misc' +import { logger } from '../../helpers/logger' +import { areValidationErrors } from './utils' +import { isValidRSSFeed } from '../../helpers/custom-validators/feeds' + +const feedsValidator = [ + param('format').optional().custom(isValidRSSFeed).withMessage('Should have a valid format (rss, atom, json)'), + query('format').optional().custom(isValidRSSFeed).withMessage('Should have a valid format (rss, atom, json)'), + query('accountId').optional().custom(isIdOrUUIDValid), + query('accountName').optional().custom(isAccountNameValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking feeds parameters', { parameters: req.query }) + + if (areValidationErrors(req, res)) return + + if (req.query.accountId) { + if (!await isAccountIdExist(req.query.accountId, res)) return + } else if (req.query.accountName) { + if (!await isLocalAccountNameExist(req.query.accountName, res)) return + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + feedsValidator +} diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts index 9840e8f65..b69e1f14b 100644 --- a/server/middlewares/validators/index.ts +++ b/server/middlewares/validators/index.ts @@ -3,6 +3,7 @@ export * from './oembed' export * from './activitypub' export * from './pagination' export * from './follows' +export * from './feeds' export * from './sort' export * from './users' export * from './videos' diff --git a/server/models/account/account.ts b/server/models/account/account.ts index c5955ef3b..3ff59887d 100644 --- a/server/models/account/account.ts +++ b/server/models/account/account.ts @@ -246,7 +246,7 @@ export class AccountModel extends Model { const actor = this.Actor.toFormattedJSON() const account = { id: this.id, - displayName: this.name, + displayName: this.getDisplayName(), description: this.description, createdAt: this.createdAt, updatedAt: this.updatedAt @@ -266,4 +266,8 @@ export class AccountModel extends Model { isOwned () { return this.Actor.isOwned() } + + getDisplayName () { + return this.name + } } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 54fe54535..240a2b5a2 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -95,14 +95,15 @@ enum ScopeNames { } @Scopes({ - [ScopeNames.AVAILABLE_FOR_LIST]: (actorId: number, filter?: VideoFilter) => ({ - where: { - id: { - [Sequelize.Op.notIn]: Sequelize.literal( - '(SELECT "videoBlacklist"."videoId" FROM "videoBlacklist")' - ), - [ Sequelize.Op.in ]: Sequelize.literal( - '(' + + [ScopeNames.AVAILABLE_FOR_LIST]: (actorId: number, filter?: VideoFilter, withFiles?: boolean) => { + const query: IFindOptions = { + where: { + id: { + [Sequelize.Op.notIn]: Sequelize.literal( + '(SELECT "videoBlacklist"."videoId" FROM "videoBlacklist")' + ), + [ Sequelize.Op.in ]: Sequelize.literal( + '(' + 'SELECT "videoShare"."videoId" AS "id" FROM "videoShare" ' + 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "videoShare"."actorId" ' + 'WHERE "actorFollow"."actorId" = ' + parseInt(actorId.toString(), 10) + @@ -113,45 +114,55 @@ enum ScopeNames { 'INNER JOIN "actor" ON "account"."actorId" = "actor"."id" ' + 'LEFT JOIN "actorFollow" ON "actorFollow"."targetActorId" = "actor"."id" ' + 'WHERE "actor"."serverId" IS NULL OR "actorFollow"."actorId" = ' + parseInt(actorId.toString(), 10) + - ')' - ) + ')' + ) + }, + privacy: VideoPrivacy.PUBLIC }, - privacy: VideoPrivacy.PUBLIC - }, - include: [ - { - attributes: [ 'name', 'description' ], - model: VideoChannelModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'name' ], - model: AccountModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'preferredUsername', 'url', 'serverId' ], - model: ActorModel.unscoped(), - required: true, - where: VideoModel.buildActorWhereWithFilter(filter), - include: [ - { - attributes: [ 'host' ], - model: ServerModel.unscoped(), - required: false - }, - { - model: AvatarModel.unscoped(), - required: false - } - ] - } - ] - } - ] - } - ] - }), + include: [ + { + attributes: [ 'name', 'description' ], + model: VideoChannelModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'name' ], + model: AccountModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'preferredUsername', 'url', 'serverId', 'avatarId' ], + model: ActorModel.unscoped(), + required: true, + where: VideoModel.buildActorWhereWithFilter(filter), + include: [ + { + attributes: [ 'host' ], + model: ServerModel.unscoped(), + required: false + }, + { + model: AvatarModel.unscoped(), + required: false + } + ] + } + ] + } + ] + } + ] + } + + if (withFiles === true) { + query.include.push({ + model: VideoFileModel.unscoped(), + required: true + }) + } + + return query + }, [ScopeNames.WITH_ACCOUNT_DETAILS]: { include: [ { @@ -629,8 +640,8 @@ export class VideoModel extends Model { }) } - static listUserVideosForApi (userId: number, start: number, count: number, sort: string) { - const query = { + static listUserVideosForApi (userId: number, start: number, count: number, sort: string, withFiles = false) { + const query: IFindOptions = { offset: start, limit: count, order: getSort(sort), @@ -651,6 +662,13 @@ export class VideoModel extends Model { ] } + if (withFiles === true) { + query.include.push({ + model: VideoFileModel.unscoped(), + required: true + }) + } + return VideoModel.findAndCountAll(query).then(({ rows, count }) => { return { data: rows, @@ -659,7 +677,7 @@ export class VideoModel extends Model { }) } - static async listForApi (start: number, count: number, sort: string, filter?: VideoFilter) { + static async listForApi (start: number, count: number, sort: string, filter?: VideoFilter, withFiles = false) { const query = { offset: start, limit: count, @@ -668,7 +686,7 @@ export class VideoModel extends Model { const serverActor = await getServerActor() - return VideoModel.scope({ method: [ ScopeNames.AVAILABLE_FOR_LIST, serverActor.id, filter ] }) + return VideoModel.scope({ method: [ ScopeNames.AVAILABLE_FOR_LIST, serverActor.id, filter, withFiles ] }) .findAndCountAll(query) .then(({ rows, count }) => { return { @@ -707,7 +725,8 @@ export class VideoModel extends Model { const serverActor = await getServerActor() return VideoModel.scope({ method: [ ScopeNames.AVAILABLE_FOR_LIST, serverActor.id ] }) - .findAndCountAll(query).then(({ rows, count }) => { + .findAndCountAll(query) + .then(({ rows, count }) => { return { data: rows, total: count @@ -1006,29 +1025,34 @@ export class VideoModel extends Model { } // Format and sort video files + detailsJson.files = this.getFormattedVideoFilesJSON() + + return Object.assign(formattedJson, detailsJson) + } + + getFormattedVideoFilesJSON (): VideoFile[] { const { baseUrlHttp, baseUrlWs } = this.getBaseUrls() - detailsJson.files = this.VideoFiles - .map(videoFile => { - let resolutionLabel = videoFile.resolution + 'p' - return { - resolution: { - id: videoFile.resolution, - label: resolutionLabel - }, - magnetUri: this.generateMagnetUri(videoFile, baseUrlHttp, baseUrlWs), - size: videoFile.size, - torrentUrl: this.getTorrentUrl(videoFile, baseUrlHttp), - fileUrl: this.getVideoFileUrl(videoFile, baseUrlHttp) - } as VideoFile - }) - .sort((a, b) => { - if (a.resolution.id < b.resolution.id) return 1 - if (a.resolution.id === b.resolution.id) return 0 - return -1 - }) + return this.VideoFiles + .map(videoFile => { + let resolutionLabel = videoFile.resolution + 'p' - return Object.assign(formattedJson, detailsJson) + return { + resolution: { + id: videoFile.resolution, + label: resolutionLabel + }, + magnetUri: this.generateMagnetUri(videoFile, baseUrlHttp, baseUrlWs), + size: videoFile.size, + torrentUrl: this.getTorrentUrl(videoFile, baseUrlHttp), + fileUrl: this.getVideoFileUrl(videoFile, baseUrlHttp) + } as VideoFile + }) + .sort((a, b) => { + if (a.resolution.id < b.resolution.id) return 1 + if (a.resolution.id === b.resolution.id) return 0 + return -1 + }) } toActivityPubObject (): VideoTorrentObject { -- cgit v1.2.3