From cb0eda5602a21d1626a7face32de6153ed07b5f9 Mon Sep 17 00:00:00 2001 From: Alecks Gates Date: Mon, 22 May 2023 09:00:05 -0500 Subject: Add Podcast RSS feeds (#5487) * Initial test implementation of Podcast RSS This is a pretty simple implementation to add support for The Podcast Namespace in RSS -- instead of affecting the existing RSS implementation, this adds a new UI option. I attempted to retain compatibility with the rest of the RSS feed implementation as much as possible and have created a temporary fork of the "pfeed" library to support this effort. * Update to pfeed-podcast 1.2.2 * Initial test implementation of Podcast RSS This is a pretty simple implementation to add support for The Podcast Namespace in RSS -- instead of affecting the existing RSS implementation, this adds a new UI option. I attempted to retain compatibility with the rest of the RSS feed implementation as much as possible and have created a temporary fork of the "pfeed" library to support this effort. * Update to pfeed-podcast 1.2.2 * Initial test implementation of Podcast RSS This is a pretty simple implementation to add support for The Podcast Namespace in RSS -- instead of affecting the existing RSS implementation, this adds a new UI option. I attempted to retain compatibility with the rest of the RSS feed implementation as much as possible and have created a temporary fork of the "pfeed" library to support this effort. * Update to pfeed-podcast 1.2.2 * Add correct feed image to RSS channel * Prefer HLS videos for podcast RSS Remove video/stream titles, add optional height attribute to podcast RSS * Prefix podcast RSS images with root server URL * Add optional video query support to include captions * Add transcripts & person images to podcast RSS feed * Prefer webseed/webtorrent files over HLS fragmented mp4s * Experimentally adding podcast fields to basic config page * Add validation for new basic config fields * Don't include "content" in podcast feed, use full description for "description" * Initial test implementation of Podcast RSS This is a pretty simple implementation to add support for The Podcast Namespace in RSS -- instead of affecting the existing RSS implementation, this adds a new UI option. I attempted to retain compatibility with the rest of the RSS feed implementation as much as possible and have created a temporary fork of the "pfeed" library to support this effort. * Update to pfeed-podcast 1.2.2 * Add correct feed image to RSS channel * Prefer HLS videos for podcast RSS Remove video/stream titles, add optional height attribute to podcast RSS * Prefix podcast RSS images with root server URL * Add optional video query support to include captions * Add transcripts & person images to podcast RSS feed * Prefer webseed/webtorrent files over HLS fragmented mp4s * Experimentally adding podcast fields to basic config page * Add validation for new basic config fields * Don't include "content" in podcast feed, use full description for "description" * Add medium/socialInteract to podcast RSS feeds. Use HTML for description * Change base production image to bullseye, install prosody in image * Add liveItem and trackers to Podcast RSS feeds Remove height from alternateEnclosure, replaced with title. * Clear Podcast RSS feed cache when live streams start/end * Upgrade to Node 16 * Refactor clearCacheRoute to use ApiCache * Remove unnecessary type hint * Update dockerfile to node 16, install python-is-python2 * Use new file paths for captions/playlists * Fix legacy videos in RSS after migration to object storage * Improve method of identifying non-fragmented mp4s in podcast RSS feeds * Don't include fragmented MP4s in podcast RSS feeds * Add experimental support for podcast:categories on the podcast RSS item * Fix undefined category when no videos exist Allows for empty feeds to exist (important for feeds that might only go live) * Add support for podcast:locked -- user has to opt in to show their email * Use comma for podcast:categories delimiter * Make cache clearing async * Fix merge, temporarily test with pfeed-podcast * Syntax changes * Add EXT_MIMETYPE constants for captions * Update & fix tests, fix enclosure mimetypes, remove admin email * Add test for podacst:socialInteract * Add filters hooks for podcast customTags * Remove showdown, updated to pfeed-podcast 6.1.2 * Add 'action:api.live-video.state.updated' hook * Avoid assigning undefined category to podcast feeds * Remove nvmrc * Remove comment * Remove unused podcast config * Remove more unused podcast config * Fix MChannelAccountDefault type hint missed in merge * Remove extra line * Re-add newline in config * Fix lint errors for isEmailPublic * Fix thumbnails in podcast feeds * Requested changes based on review * Provide podcast rss 2.0 only on video channels * Misc cleanup for a less messy PR * Lint fixes * Remove pfeed-podcast * Add peertube version to new hooks * Don't use query include, remove TODO * Remove film medium hack * Clear podcast rss cache before video/channel update hooks * Clear podcast rss cache before video uploaded/deleted hooks * Refactor podcast feed cache clearing * Set correct person name from video channel * Styling * Fix tests --------- Co-authored-by: Chocobozzz --- .../controllers/feeds/shared/common-feed-utils.ts | 145 +++++++++++++++++++++ server/controllers/feeds/shared/index.ts | 2 + .../controllers/feeds/shared/video-feed-utils.ts | 66 ++++++++++ 3 files changed, 213 insertions(+) create mode 100644 server/controllers/feeds/shared/common-feed-utils.ts create mode 100644 server/controllers/feeds/shared/index.ts create mode 100644 server/controllers/feeds/shared/video-feed-utils.ts (limited to 'server/controllers/feeds/shared') diff --git a/server/controllers/feeds/shared/common-feed-utils.ts b/server/controllers/feeds/shared/common-feed-utils.ts new file mode 100644 index 000000000..375c2814b --- /dev/null +++ b/server/controllers/feeds/shared/common-feed-utils.ts @@ -0,0 +1,145 @@ +import express from 'express' +import { Feed } from '@peertube/feed' +import { CustomTag, CustomXMLNS, Person } from '@peertube/feed/lib/typings' +import { mdToOneLinePlainText } from '@server/helpers/markdown' +import { CONFIG } from '@server/initializers/config' +import { WEBSERVER } from '@server/initializers/constants' +import { UserModel } from '@server/models/user/user' +import { MAccountDefault, MChannelBannerAccountDefault, MUser, MVideoFullLight } from '@server/types/models' +import { pick } from '@shared/core-utils' +import { ActorImageType } from '@shared/models' + +export function initFeed (parameters: { + name: string + description: string + imageUrl: string + isPodcast: boolean + link?: string + locked?: { isLocked: boolean, email: string } + author?: { + name: string + link: string + imageUrl: string + } + person?: Person[] + resourceType?: 'videos' | 'video-comments' + queryString?: string + medium?: string + stunServers?: string[] + trackers?: string[] + customXMLNS?: CustomXMLNS[] + customTags?: CustomTag[] +}) { + const webserverUrl = WEBSERVER.URL + const { name, description, link, imageUrl, isPodcast, resourceType, queryString, medium } = parameters + + return new Feed({ + title: name, + description: mdToOneLinePlainText(description), + // updated: TODO: somehowGetLatestUpdate, // optional, default = today + id: link || webserverUrl, + link: link || webserverUrl, + image: imageUrl, + favicon: webserverUrl + '/client/assets/images/favicon.png', + copyright: `All rights reserved, unless otherwise specified in the terms specified at ${webserverUrl}/about` + + ` and potential licenses granted by each content's rightholder.`, + generator: `Toraifōsu`, // ^.~ + medium: medium || 'video', + feedLinks: { + json: `${webserverUrl}/feeds/${resourceType}.json${queryString}`, + atom: `${webserverUrl}/feeds/${resourceType}.atom${queryString}`, + rss: isPodcast + ? `${webserverUrl}/feeds/podcast/videos.xml${queryString}` + : `${webserverUrl}/feeds/${resourceType}.xml${queryString}` + }, + + ...pick(parameters, [ 'stunServers', 'trackers', 'customXMLNS', 'customTags', 'author', 'person', 'locked' ]) + }) +} + +export function sendFeed (feed: Feed, req: express.Request, res: express.Response) { + const format = req.params.format + + if (format === 'atom' || format === 'atom1') { + return res.send(feed.atom1()).end() + } + + if (format === 'json' || format === 'json1') { + return res.send(feed.json1()).end() + } + + if (format === 'rss' || format === 'rss2') { + return res.send(feed.rss2()).end() + } + + // We're in the ambiguous '.xml' case and we look at the format query parameter + if (req.query.format === 'atom' || req.query.format === 'atom1') { + return res.send(feed.atom1()).end() + } + + return res.send(feed.rss2()).end() +} + +export async function buildFeedMetadata (options: { + videoChannel?: MChannelBannerAccountDefault + account?: MAccountDefault + video?: MVideoFullLight +}) { + const { video, videoChannel, account } = options + + let imageUrl = WEBSERVER.URL + '/client/assets/images/icons/icon-96x96.png' + let accountImageUrl: string + let name: string + let userName: string + let description: string + let email: string + let link: string + let accountLink: string + let user: MUser + + if (videoChannel) { + name = videoChannel.getDisplayName() + description = videoChannel.description + link = videoChannel.getClientUrl() + accountLink = videoChannel.Account.getClientUrl() + + if (videoChannel.Actor.hasImage(ActorImageType.AVATAR)) { + imageUrl = WEBSERVER.URL + videoChannel.Actor.Avatars[0].getStaticPath() + } + + if (videoChannel.Account.Actor.hasImage(ActorImageType.AVATAR)) { + accountImageUrl = WEBSERVER.URL + videoChannel.Account.Actor.Avatars[0].getStaticPath() + } + + user = await UserModel.loadById(videoChannel.Account.userId) + userName = videoChannel.Account.getDisplayName() + } else if (account) { + name = account.getDisplayName() + description = account.description + link = account.getClientUrl() + accountLink = link + + if (account.Actor.hasImage(ActorImageType.AVATAR)) { + imageUrl = WEBSERVER.URL + account.Actor.Avatars[0].getStaticPath() + accountImageUrl = imageUrl + } + + user = await UserModel.loadById(account.userId) + } else if (video) { + name = video.name + description = video.description + link = video.url + } else { + name = CONFIG.INSTANCE.NAME + description = CONFIG.INSTANCE.DESCRIPTION + link = WEBSERVER.URL + } + + // If the user is local, has a verified email address, and allows it to be publicly displayed + // Return it so the owner can prove ownership of their feed + if (user && !user.pluginAuth && user.emailVerified && user.emailPublic) { + email = user.email + } + + return { name, userName, description, imageUrl, accountImageUrl, email, link, accountLink } +} diff --git a/server/controllers/feeds/shared/index.ts b/server/controllers/feeds/shared/index.ts new file mode 100644 index 000000000..0136c8477 --- /dev/null +++ b/server/controllers/feeds/shared/index.ts @@ -0,0 +1,2 @@ +export * from './video-feed-utils' +export * from './common-feed-utils' diff --git a/server/controllers/feeds/shared/video-feed-utils.ts b/server/controllers/feeds/shared/video-feed-utils.ts new file mode 100644 index 000000000..3175cea59 --- /dev/null +++ b/server/controllers/feeds/shared/video-feed-utils.ts @@ -0,0 +1,66 @@ +import { mdToOneLinePlainText, toSafeHtml } from '@server/helpers/markdown' +import { CONFIG } from '@server/initializers/config' +import { WEBSERVER } from '@server/initializers/constants' +import { getServerActor } from '@server/models/application/application' +import { getCategoryLabel } from '@server/models/video/formatter/video-format-utils' +import { DisplayOnlyForFollowerOptions } from '@server/models/video/sql/video' +import { VideoModel } from '@server/models/video/video' +import { MThumbnail, MUserDefault } from '@server/types/models' +import { VideoInclude } from '@shared/models' + +export async function getVideosForFeeds (options: { + sort: string + nsfw: boolean + isLocal: boolean + include: VideoInclude + + accountId?: number + videoChannelId?: number + displayOnlyForFollower?: DisplayOnlyForFollowerOptions + user?: MUserDefault +}) { + const server = await getServerActor() + + const { data } = await VideoModel.listForApi({ + start: 0, + count: CONFIG.FEEDS.VIDEOS.COUNT, + displayOnlyForFollower: { + actorId: server.id, + orLocalVideos: true + }, + hasFiles: true, + countVideos: false, + + ...options + }) + + return data +} + +export function getCommonVideoFeedAttributes (video: VideoModel) { + const localLink = WEBSERVER.URL + video.getWatchStaticPath() + + const thumbnailModels: MThumbnail[] = [] + if (video.hasPreview()) thumbnailModels.push(video.getPreview()) + thumbnailModels.push(video.getMiniature()) + + return { + title: video.name, + link: localLink, + description: mdToOneLinePlainText(video.getTruncatedDescription()), + content: toSafeHtml(video.description), + + date: video.publishedAt, + nsfw: video.nsfw, + + category: video.category + ? [ { name: getCategoryLabel(video.category) } ] + : undefined, + + thumbnails: thumbnailModels.map(t => ({ + url: WEBSERVER.URL + t.getLocalStaticPath(), + width: t.width, + height: t.height + })) + } +} -- cgit v1.2.3