From cb0eda5602a21d1626a7face32de6153ed07b5f9 Mon Sep 17 00:00:00 2001 From: Alecks Gates Date: Mon, 22 May 2023 09:00:05 -0500 Subject: Add Podcast RSS feeds (#5487) * Initial test implementation of Podcast RSS This is a pretty simple implementation to add support for The Podcast Namespace in RSS -- instead of affecting the existing RSS implementation, this adds a new UI option. I attempted to retain compatibility with the rest of the RSS feed implementation as much as possible and have created a temporary fork of the "pfeed" library to support this effort. * Update to pfeed-podcast 1.2.2 * Initial test implementation of Podcast RSS This is a pretty simple implementation to add support for The Podcast Namespace in RSS -- instead of affecting the existing RSS implementation, this adds a new UI option. I attempted to retain compatibility with the rest of the RSS feed implementation as much as possible and have created a temporary fork of the "pfeed" library to support this effort. * Update to pfeed-podcast 1.2.2 * Initial test implementation of Podcast RSS This is a pretty simple implementation to add support for The Podcast Namespace in RSS -- instead of affecting the existing RSS implementation, this adds a new UI option. I attempted to retain compatibility with the rest of the RSS feed implementation as much as possible and have created a temporary fork of the "pfeed" library to support this effort. * Update to pfeed-podcast 1.2.2 * Add correct feed image to RSS channel * Prefer HLS videos for podcast RSS Remove video/stream titles, add optional height attribute to podcast RSS * Prefix podcast RSS images with root server URL * Add optional video query support to include captions * Add transcripts & person images to podcast RSS feed * Prefer webseed/webtorrent files over HLS fragmented mp4s * Experimentally adding podcast fields to basic config page * Add validation for new basic config fields * Don't include "content" in podcast feed, use full description for "description" * Initial test implementation of Podcast RSS This is a pretty simple implementation to add support for The Podcast Namespace in RSS -- instead of affecting the existing RSS implementation, this adds a new UI option. I attempted to retain compatibility with the rest of the RSS feed implementation as much as possible and have created a temporary fork of the "pfeed" library to support this effort. * Update to pfeed-podcast 1.2.2 * Add correct feed image to RSS channel * Prefer HLS videos for podcast RSS Remove video/stream titles, add optional height attribute to podcast RSS * Prefix podcast RSS images with root server URL * Add optional video query support to include captions * Add transcripts & person images to podcast RSS feed * Prefer webseed/webtorrent files over HLS fragmented mp4s * Experimentally adding podcast fields to basic config page * Add validation for new basic config fields * Don't include "content" in podcast feed, use full description for "description" * Add medium/socialInteract to podcast RSS feeds. Use HTML for description * Change base production image to bullseye, install prosody in image * Add liveItem and trackers to Podcast RSS feeds Remove height from alternateEnclosure, replaced with title. * Clear Podcast RSS feed cache when live streams start/end * Upgrade to Node 16 * Refactor clearCacheRoute to use ApiCache * Remove unnecessary type hint * Update dockerfile to node 16, install python-is-python2 * Use new file paths for captions/playlists * Fix legacy videos in RSS after migration to object storage * Improve method of identifying non-fragmented mp4s in podcast RSS feeds * Don't include fragmented MP4s in podcast RSS feeds * Add experimental support for podcast:categories on the podcast RSS item * Fix undefined category when no videos exist Allows for empty feeds to exist (important for feeds that might only go live) * Add support for podcast:locked -- user has to opt in to show their email * Use comma for podcast:categories delimiter * Make cache clearing async * Fix merge, temporarily test with pfeed-podcast * Syntax changes * Add EXT_MIMETYPE constants for captions * Update & fix tests, fix enclosure mimetypes, remove admin email * Add test for podacst:socialInteract * Add filters hooks for podcast customTags * Remove showdown, updated to pfeed-podcast 6.1.2 * Add 'action:api.live-video.state.updated' hook * Avoid assigning undefined category to podcast feeds * Remove nvmrc * Remove comment * Remove unused podcast config * Remove more unused podcast config * Fix MChannelAccountDefault type hint missed in merge * Remove extra line * Re-add newline in config * Fix lint errors for isEmailPublic * Fix thumbnails in podcast feeds * Requested changes based on review * Provide podcast rss 2.0 only on video channels * Misc cleanup for a less messy PR * Lint fixes * Remove pfeed-podcast * Add peertube version to new hooks * Don't use query include, remove TODO * Remove film medium hack * Clear podcast rss cache before video/channel update hooks * Clear podcast rss cache before video uploaded/deleted hooks * Refactor podcast feed cache clearing * Set correct person name from video channel * Styling * Fix tests --------- Co-authored-by: Chocobozzz --- server/middlewares/cache/cache.ts | 14 +++++++-- server/middlewares/cache/shared/api-cache.ts | 45 ++++++++++++++++++++++++--- server/middlewares/validators/feeds.ts | 46 ++++++++++++++++++++++++++++ server/middlewares/validators/users.ts | 4 +++ 4 files changed, 103 insertions(+), 6 deletions(-) (limited to 'server/middlewares') diff --git a/server/middlewares/cache/cache.ts b/server/middlewares/cache/cache.ts index e14160ba8..6041c76c3 100644 --- a/server/middlewares/cache/cache.ts +++ b/server/middlewares/cache/cache.ts @@ -17,12 +17,22 @@ function cacheRoute (duration: string) { function cacheRouteFactory (options: APICacheOptions) { const instance = new ApiCache({ ...defaultOptions, ...options }) - return instance.buildMiddleware.bind(instance) + return { instance, middleware: instance.buildMiddleware.bind(instance) } +} + +// --------------------------------------------------------------------------- + +function buildPodcastGroupsCache (options: { + channelId: number +}) { + return 'podcast-feed-' + options.channelId } // --------------------------------------------------------------------------- export { cacheRoute, - cacheRouteFactory + cacheRouteFactory, + + buildPodcastGroupsCache } diff --git a/server/middlewares/cache/shared/api-cache.ts b/server/middlewares/cache/shared/api-cache.ts index 7c366db00..c6197b972 100644 --- a/server/middlewares/cache/shared/api-cache.ts +++ b/server/middlewares/cache/shared/api-cache.ts @@ -27,7 +27,13 @@ export class ApiCache { private readonly options: APICacheOptions private readonly timers: { [ id: string ]: NodeJS.Timeout } = {} - private readonly index: { all: string[] } = { all: [] } + private readonly index = { + groups: [] as string[], + all: [] as string[] + } + + // Cache keys per group + private groups: { [groupIndex: string]: string[] } = {} constructor (options: APICacheOptions) { this.options = { @@ -43,7 +49,7 @@ export class ApiCache { return asyncMiddleware( async (req: express.Request, res: express.Response, next: express.NextFunction) => { - const key = Redis.Instance.getPrefix() + 'api-cache-' + req.originalUrl + const key = this.getCacheKey(req) const redis = Redis.Instance.getClient() if (!Redis.Instance.isConnected()) return this.makeResponseCacheable(res, next, key, duration) @@ -62,6 +68,29 @@ export class ApiCache { ) } + clearGroupSafe (group: string) { + const run = async () => { + const cacheKeys = this.groups[group] + if (!cacheKeys) return + + for (const key of cacheKeys) { + try { + await this.clear(key) + } catch (err) { + logger.error('Cannot clear ' + key, { err }) + } + } + + delete this.groups[group] + } + + void run() + } + + private getCacheKey (req: express.Request) { + return Redis.Instance.getPrefix() + 'api-cache-' + req.originalUrl + } + private shouldCacheResponse (response: express.Response) { if (!response) return false if (this.options.excludeStatus.includes(response.statusCode)) return false @@ -69,8 +98,16 @@ export class ApiCache { return true } - private addIndexEntries (key: string) { + private addIndexEntries (key: string, res: express.Response) { this.index.all.unshift(key) + + const groups = res.locals.apicacheGroups || [] + + for (const group of groups) { + if (!this.groups[group]) this.groups[group] = [] + + this.groups[group].push(key) + } } private filterBlacklistedHeaders (headers: OutgoingHttpHeaders) { @@ -177,7 +214,7 @@ export class ApiCache { self.accumulateContent(res, content) if (res.locals.apicache.cacheable && res.locals.apicache.content) { - self.addIndexEntries(key) + self.addIndexEntries(key, res) const headers = res.locals.apicache.headers || res.getHeaders() const cacheObject = self.createCacheObject( diff --git a/server/middlewares/validators/feeds.ts b/server/middlewares/validators/feeds.ts index 0bfe89e6f..ee8615cae 100644 --- a/server/middlewares/validators/feeds.ts +++ b/server/middlewares/validators/feeds.ts @@ -3,6 +3,7 @@ import { param, query } from 'express-validator' import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' import { isValidRSSFeed } from '../../helpers/custom-validators/feeds' import { exists, isIdOrUUIDValid, isIdValid, toCompleteUUID } from '../../helpers/custom-validators/misc' +import { buildPodcastGroupsCache } from '../cache' import { areValidationErrors, checkCanSeeVideo, @@ -43,6 +44,21 @@ function setFeedFormatContentType (req: express.Request, res: express.Response, acceptableContentTypes = [ 'application/xml', 'text/xml' ] } + return feedContentTypeResponse(req, res, next, acceptableContentTypes) +} + +function setFeedPodcastContentType (req: express.Request, res: express.Response, next: express.NextFunction) { + const acceptableContentTypes = [ 'application/rss+xml', 'application/xml', 'text/xml' ] + + return feedContentTypeResponse(req, res, next, acceptableContentTypes) +} + +function feedContentTypeResponse ( + req: express.Request, + res: express.Response, + next: express.NextFunction, + acceptableContentTypes: string[] +) { if (req.accepts(acceptableContentTypes)) { res.set('Content-Type', req.accepts(acceptableContentTypes) as string) } else { @@ -55,6 +71,8 @@ function setFeedFormatContentType (req: express.Request, res: express.Response, return next() } +// --------------------------------------------------------------------------- + const videoFeedsValidator = [ query('accountId') .optional() @@ -82,6 +100,31 @@ const videoFeedsValidator = [ } ] +// --------------------------------------------------------------------------- + +const videoFeedsPodcastValidator = [ + query('videoChannelId') + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoChannelIdExist(req.query.videoChannelId, res)) return + + return next() + } +] + +const videoFeedsPodcastSetCacheKey = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (req.query.videoChannelId) { + res.locals.apicacheGroups = [ buildPodcastGroupsCache({ channelId: req.query.videoChannelId }) ] + } + + return next() + } +] +// --------------------------------------------------------------------------- + const videoSubscriptionFeedsValidator = [ query('accountId') .custom(isIdValid), @@ -126,7 +169,10 @@ const videoCommentsFeedsValidator = [ export { feedsFormatValidator, setFeedFormatContentType, + setFeedPodcastContentType, videoFeedsValidator, + videoFeedsPodcastValidator, videoSubscriptionFeedsValidator, + videoFeedsPodcastSetCacheKey, videoCommentsFeedsValidator } diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts index 7ebea048d..3d311b15b 100644 --- a/server/middlewares/validators/users.ts +++ b/server/middlewares/validators/users.ts @@ -11,6 +11,7 @@ import { isUserBlockedReasonValid, isUserDescriptionValid, isUserDisplayNameValid, + isUserEmailPublicValid, isUserNoModal, isUserNSFWPolicyValid, isUserP2PEnabledValid, @@ -213,6 +214,9 @@ const usersUpdateMeValidator = [ body('password') .optional() .custom(isUserPasswordValid), + body('emailPublic') + .optional() + .custom(isUserEmailPublicValid), body('email') .optional() .isEmail(), -- cgit v1.2.3