Add Podcast RSS feeds (#5487)

* Initial test implementation of Podcast RSS

This is a pretty simple implementation to add support for The Podcast Namespace in RSS -- instead of affecting the existing RSS implementation, this adds a new UI option.

I attempted to retain compatibility with the rest of the RSS feed implementation as much as possible and have created a temporary fork of the "pfeed" library to support this effort.

* Update to pfeed-podcast 1.2.2

* Initial test implementation of Podcast RSS

This is a pretty simple implementation to add support for The Podcast Namespace in RSS -- instead of affecting the existing RSS implementation, this adds a new UI option.

I attempted to retain compatibility with the rest of the RSS feed implementation as much as possible and have created a temporary fork of the "pfeed" library to support this effort.

* Update to pfeed-podcast 1.2.2

* Initial test implementation of Podcast RSS

This is a pretty simple implementation to add support for The Podcast Namespace in RSS -- instead of affecting the existing RSS implementation, this adds a new UI option.

I attempted to retain compatibility with the rest of the RSS feed implementation as much as possible and have created a temporary fork of the "pfeed" library to support this effort.

* Update to pfeed-podcast 1.2.2

* Add correct feed image to RSS channel

* Prefer HLS videos for podcast RSS

Remove video/stream titles, add optional height attribute to podcast RSS

* Prefix podcast RSS images with root server URL

* Add optional video query support to include captions

* Add transcripts & person images to podcast RSS feed

* Prefer webseed/webtorrent files over HLS fragmented mp4s

* Experimentally adding podcast fields to basic config page

* Add validation for new basic config fields

* Don't include "content" in podcast feed, use full description for "description"

* Initial test implementation of Podcast RSS

This is a pretty simple implementation to add support for The Podcast Namespace in RSS -- instead of affecting the existing RSS implementation, this adds a new UI option.

I attempted to retain compatibility with the rest of the RSS feed implementation as much as possible and have created a temporary fork of the "pfeed" library to support this effort.

* Update to pfeed-podcast 1.2.2

* Add correct feed image to RSS channel

* Prefer HLS videos for podcast RSS

Remove video/stream titles, add optional height attribute to podcast RSS

* Prefix podcast RSS images with root server URL

* Add optional video query support to include captions

* Add transcripts & person images to podcast RSS feed

* Prefer webseed/webtorrent files over HLS fragmented mp4s

* Experimentally adding podcast fields to basic config page

* Add validation for new basic config fields

* Don't include "content" in podcast feed, use full description for "description"

* Add medium/socialInteract to podcast RSS feeds. Use HTML for description

* Change base production image to bullseye, install prosody in image

* Add liveItem and trackers to Podcast RSS feeds

Remove height from alternateEnclosure, replaced with title.

* Clear Podcast RSS feed cache when live streams start/end

* Upgrade to Node 16

* Refactor clearCacheRoute to use ApiCache

* Remove unnecessary type hint

* Update dockerfile to node 16, install python-is-python2

* Use new file paths for captions/playlists

* Fix legacy videos in RSS after migration to object storage

* Improve method of identifying non-fragmented mp4s in podcast RSS feeds

* Don't include fragmented MP4s in podcast RSS feeds

* Add experimental support for podcast:categories on the podcast RSS item

* Fix undefined category when no videos exist

Allows for empty feeds to exist (important for feeds that might only go live)

* Add support for podcast:locked -- user has to opt in to show their email

* Use comma for podcast:categories delimiter

* Make cache clearing async

* Fix merge, temporarily test with pfeed-podcast

* Syntax changes

* Add EXT_MIMETYPE constants for captions

* Update & fix tests, fix enclosure mimetypes, remove admin email

* Add test for podacst:socialInteract

* Add filters hooks for podcast customTags

* Remove showdown, updated to pfeed-podcast 6.1.2

* Add 'action:api.live-video.state.updated' hook

* Avoid assigning undefined category to podcast feeds

* Remove nvmrc

* Remove comment

* Remove unused podcast config

* Remove more unused podcast config

* Fix MChannelAccountDefault type hint missed in merge

* Remove extra line

* Re-add newline in config

* Fix lint errors for isEmailPublic

* Fix thumbnails in podcast feeds

* Requested changes based on review

* Provide podcast rss 2.0 only on video channels

* Misc cleanup for a less messy PR

* Lint fixes

* Remove pfeed-podcast

* Add peertube version to new hooks

* Don't use query include, remove TODO

* Remove film medium hack

* Clear podcast rss cache before video/channel update hooks

* Clear podcast rss cache before video uploaded/deleted hooks

* Refactor podcast feed cache clearing

* Set correct person name from video channel

* Styling

* Fix tests

---------

Co-authored-by: Chocobozzz <me@florianbigard.com>

4 files changed, 103 insertions, 6 deletions

diff --git a/server/middlewares/cache/cache.ts b/server/middlewares/cache/cache.ts
index e14160ba8..6041c76c3 100644
--- a/server/middlewares/cache/cache.ts
+++ b/server/middlewares/cache/cache.ts
@@ -17,12 +17,22 @@ function cacheRoute (duration: string) {
 function cacheRouteFactory (options: APICacheOptions) {
   const instance = new ApiCache({ ...defaultOptions, ...options })
 
-  return instance.buildMiddleware.bind(instance)
+  return { instance, middleware: instance.buildMiddleware.bind(instance) }
+}
+
+// ---------------------------------------------------------------------------
+
+function buildPodcastGroupsCache (options: {
+  channelId: number
+}) {
+  return 'podcast-feed-' + options.channelId
 }
 
 // ---------------------------------------------------------------------------
 
 export {
   cacheRoute,
-  cacheRouteFactory
+  cacheRouteFactory,
+
+  buildPodcastGroupsCache
 }
diff --git a/server/middlewares/cache/shared/api-cache.ts b/server/middlewares/cache/shared/api-cache.ts
index 7c366db00..c6197b972 100644
--- a/server/middlewares/cache/shared/api-cache.ts
+++ b/server/middlewares/cache/shared/api-cache.ts
@@ -27,7 +27,13 @@ export class ApiCache {
   private readonly options: APICacheOptions
   private readonly timers: { [ id: string ]: NodeJS.Timeout } = {}
 
-  private readonly index: { all: string[] } = { all: [] }
+  private readonly index = {
+    groups: [] as string[],
+    all: [] as string[]
+  }
+
+  // Cache keys per group
+  private groups: { [groupIndex: string]: string[] } = {}
 
   constructor (options: APICacheOptions) {
     this.options = {
@@ -43,7 +49,7 @@ export class ApiCache {
 
     return asyncMiddleware(
       async (req: express.Request, res: express.Response, next: express.NextFunction) => {
-        const key = Redis.Instance.getPrefix() + 'api-cache-' + req.originalUrl
+        const key = this.getCacheKey(req)
         const redis = Redis.Instance.getClient()
 
         if (!Redis.Instance.isConnected()) return this.makeResponseCacheable(res, next, key, duration)
@@ -62,6 +68,29 @@ export class ApiCache {
     )
   }
 
+  clearGroupSafe (group: string) {
+    const run = async () => {
+      const cacheKeys = this.groups[group]
+      if (!cacheKeys) return
+
+      for (const key of cacheKeys) {
+        try {
+          await this.clear(key)
+        } catch (err) {
+          logger.error('Cannot clear ' + key, { err })
+        }
+      }
+
+      delete this.groups[group]
+    }
+
+    void run()
+  }
+
+  private getCacheKey (req: express.Request) {
+    return Redis.Instance.getPrefix() + 'api-cache-' + req.originalUrl
+  }
+
   private shouldCacheResponse (response: express.Response) {
     if (!response) return false
     if (this.options.excludeStatus.includes(response.statusCode)) return false
@@ -69,8 +98,16 @@ export class ApiCache {
     return true
   }
 
-  private addIndexEntries (key: string) {
+  private addIndexEntries (key: string, res: express.Response) {
     this.index.all.unshift(key)
+
+    const groups = res.locals.apicacheGroups || []
+
+    for (const group of groups) {
+      if (!this.groups[group]) this.groups[group] = []
+
+      this.groups[group].push(key)
+    }
   }
 
   private filterBlacklistedHeaders (headers: OutgoingHttpHeaders) {
@@ -177,7 +214,7 @@ export class ApiCache {
         self.accumulateContent(res, content)
 
         if (res.locals.apicache.cacheable && res.locals.apicache.content) {
-          self.addIndexEntries(key)
+          self.addIndexEntries(key, res)
 
           const headers = res.locals.apicache.headers || res.getHeaders()
           const cacheObject = self.createCacheObject(
diff --git a/server/middlewares/validators/feeds.ts b/server/middlewares/validators/feeds.ts
index 0bfe89e6f..ee8615cae 100644
--- a/server/middlewares/validators/feeds.ts
+++ b/server/middlewares/validators/feeds.ts
@@ -3,6 +3,7 @@ import { param, query } from 'express-validator'
 import { HttpStatusCode } from '../../../shared/models/http/http-error-codes'
 import { isValidRSSFeed } from '../../helpers/custom-validators/feeds'
 import { exists, isIdOrUUIDValid, isIdValid, toCompleteUUID } from '../../helpers/custom-validators/misc'
+import { buildPodcastGroupsCache } from '../cache'
 import {
   areValidationErrors,
   checkCanSeeVideo,
@@ -43,6 +44,21 @@ function setFeedFormatContentType (req: express.Request, res: express.Response,
     acceptableContentTypes = [ 'application/xml', 'text/xml' ]
   }
 
+  return feedContentTypeResponse(req, res, next, acceptableContentTypes)
+}
+
+function setFeedPodcastContentType (req: express.Request, res: express.Response, next: express.NextFunction) {
+  const acceptableContentTypes = [ 'application/rss+xml', 'application/xml', 'text/xml' ]
+
+  return feedContentTypeResponse(req, res, next, acceptableContentTypes)
+}
+
+function feedContentTypeResponse (
+  req: express.Request,
+  res: express.Response,
+  next: express.NextFunction,
+  acceptableContentTypes: string[]
+) {
   if (req.accepts(acceptableContentTypes)) {
     res.set('Content-Type', req.accepts(acceptableContentTypes) as string)
   } else {
@@ -55,6 +71,8 @@ function setFeedFormatContentType (req: express.Request, res: express.Response,
   return next()
 }
 
+// ---------------------------------------------------------------------------
+
 const videoFeedsValidator = [
   query('accountId')
     .optional()
@@ -82,6 +100,31 @@ const videoFeedsValidator = [
   }
 ]
 
+// ---------------------------------------------------------------------------
+
+const videoFeedsPodcastValidator = [
+  query('videoChannelId')
+    .custom(isIdValid),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res)) return
+    if (!await doesVideoChannelIdExist(req.query.videoChannelId, res)) return
+
+    return next()
+  }
+]
+
+const videoFeedsPodcastSetCacheKey = [
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (req.query.videoChannelId) {
+      res.locals.apicacheGroups = [ buildPodcastGroupsCache({ channelId: req.query.videoChannelId }) ]
+    }
+
+    return next()
+  }
+]
+// ---------------------------------------------------------------------------
+
 const videoSubscriptionFeedsValidator = [
   query('accountId')
     .custom(isIdValid),
@@ -126,7 +169,10 @@ const videoCommentsFeedsValidator = [
 export {
   feedsFormatValidator,
   setFeedFormatContentType,
+  setFeedPodcastContentType,
   videoFeedsValidator,
+  videoFeedsPodcastValidator,
   videoSubscriptionFeedsValidator,
+  videoFeedsPodcastSetCacheKey,
   videoCommentsFeedsValidator
 }
diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts
index 7ebea048d..3d311b15b 100644
--- a/server/middlewares/validators/users.ts
+++ b/server/middlewares/validators/users.ts
@@ -11,6 +11,7 @@ import {
   isUserBlockedReasonValid,
   isUserDescriptionValid,
   isUserDisplayNameValid,
+  isUserEmailPublicValid,
   isUserNoModal,
   isUserNSFWPolicyValid,
   isUserP2PEnabledValid,
@@ -213,6 +214,9 @@ const usersUpdateMeValidator = [
   body('password')
     .optional()
     .custom(isUserPasswordValid),
+  body('emailPublic')
+    .optional()
+    .custom(isUserEmailPublicValid),
   body('email')
     .optional()
     .isEmail(),