From cb0eda5602a21d1626a7face32de6153ed07b5f9 Mon Sep 17 00:00:00 2001 From: Alecks Gates Date: Mon, 22 May 2023 09:00:05 -0500 Subject: Add Podcast RSS feeds (#5487) * Initial test implementation of Podcast RSS This is a pretty simple implementation to add support for The Podcast Namespace in RSS -- instead of affecting the existing RSS implementation, this adds a new UI option. I attempted to retain compatibility with the rest of the RSS feed implementation as much as possible and have created a temporary fork of the "pfeed" library to support this effort. * Update to pfeed-podcast 1.2.2 * Initial test implementation of Podcast RSS This is a pretty simple implementation to add support for The Podcast Namespace in RSS -- instead of affecting the existing RSS implementation, this adds a new UI option. I attempted to retain compatibility with the rest of the RSS feed implementation as much as possible and have created a temporary fork of the "pfeed" library to support this effort. * Update to pfeed-podcast 1.2.2 * Initial test implementation of Podcast RSS This is a pretty simple implementation to add support for The Podcast Namespace in RSS -- instead of affecting the existing RSS implementation, this adds a new UI option. I attempted to retain compatibility with the rest of the RSS feed implementation as much as possible and have created a temporary fork of the "pfeed" library to support this effort. * Update to pfeed-podcast 1.2.2 * Add correct feed image to RSS channel * Prefer HLS videos for podcast RSS Remove video/stream titles, add optional height attribute to podcast RSS * Prefix podcast RSS images with root server URL * Add optional video query support to include captions * Add transcripts & person images to podcast RSS feed * Prefer webseed/webtorrent files over HLS fragmented mp4s * Experimentally adding podcast fields to basic config page * Add validation for new basic config fields * Don't include "content" in podcast feed, use full description for "description" * Initial test implementation of Podcast RSS This is a pretty simple implementation to add support for The Podcast Namespace in RSS -- instead of affecting the existing RSS implementation, this adds a new UI option. I attempted to retain compatibility with the rest of the RSS feed implementation as much as possible and have created a temporary fork of the "pfeed" library to support this effort. * Update to pfeed-podcast 1.2.2 * Add correct feed image to RSS channel * Prefer HLS videos for podcast RSS Remove video/stream titles, add optional height attribute to podcast RSS * Prefix podcast RSS images with root server URL * Add optional video query support to include captions * Add transcripts & person images to podcast RSS feed * Prefer webseed/webtorrent files over HLS fragmented mp4s * Experimentally adding podcast fields to basic config page * Add validation for new basic config fields * Don't include "content" in podcast feed, use full description for "description" * Add medium/socialInteract to podcast RSS feeds. Use HTML for description * Change base production image to bullseye, install prosody in image * Add liveItem and trackers to Podcast RSS feeds Remove height from alternateEnclosure, replaced with title. * Clear Podcast RSS feed cache when live streams start/end * Upgrade to Node 16 * Refactor clearCacheRoute to use ApiCache * Remove unnecessary type hint * Update dockerfile to node 16, install python-is-python2 * Use new file paths for captions/playlists * Fix legacy videos in RSS after migration to object storage * Improve method of identifying non-fragmented mp4s in podcast RSS feeds * Don't include fragmented MP4s in podcast RSS feeds * Add experimental support for podcast:categories on the podcast RSS item * Fix undefined category when no videos exist Allows for empty feeds to exist (important for feeds that might only go live) * Add support for podcast:locked -- user has to opt in to show their email * Use comma for podcast:categories delimiter * Make cache clearing async * Fix merge, temporarily test with pfeed-podcast * Syntax changes * Add EXT_MIMETYPE constants for captions * Update & fix tests, fix enclosure mimetypes, remove admin email * Add test for podacst:socialInteract * Add filters hooks for podcast customTags * Remove showdown, updated to pfeed-podcast 6.1.2 * Add 'action:api.live-video.state.updated' hook * Avoid assigning undefined category to podcast feeds * Remove nvmrc * Remove comment * Remove unused podcast config * Remove more unused podcast config * Fix MChannelAccountDefault type hint missed in merge * Remove extra line * Re-add newline in config * Fix lint errors for isEmailPublic * Fix thumbnails in podcast feeds * Requested changes based on review * Provide podcast rss 2.0 only on video channels * Misc cleanup for a less messy PR * Lint fixes * Remove pfeed-podcast * Add peertube version to new hooks * Don't use query include, remove TODO * Remove film medium hack * Clear podcast rss cache before video/channel update hooks * Clear podcast rss cache before video uploaded/deleted hooks * Refactor podcast feed cache clearing * Set correct person name from video channel * Styling * Fix tests --------- Co-authored-by: Chocobozzz --- server/models/account/account.ts | 12 ++--- server/models/actor/actor.ts | 8 ++-- server/models/user/user.ts | 6 +++ .../models/video/formatter/video-format-utils.ts | 2 +- server/models/video/thumbnail.ts | 6 ++- server/models/video/video-caption.ts | 27 ++++++++++- server/models/video/video-channel.ts | 27 +++++++++-- server/models/video/video.ts | 53 +++++++++++++++++----- 8 files changed, 112 insertions(+), 29 deletions(-) (limited to 'server/models') diff --git a/server/models/account/account.ts b/server/models/account/account.ts index ec4e8d946..396959352 100644 --- a/server/models/account/account.ts +++ b/server/models/account/account.ts @@ -28,8 +28,9 @@ import { MAccountAP, MAccountDefault, MAccountFormattable, + MAccountHost, MAccountSummaryFormattable, - MChannelActor + MChannelHost } from '../../types/models' import { ActorModel } from '../actor/actor' import { ActorFollowModel } from '../actor/actor-follow' @@ -410,10 +411,6 @@ export class AccountModel extends Model>> { .findAll(query) } - getClientUrl () { - return WEBSERVER.URL + '/accounts/' + this.Actor.getIdentifier() - } - toFormattedJSON (this: MAccountFormattable): Account { return { ...this.Actor.toFormattedJSON(), @@ -463,8 +460,9 @@ export class AccountModel extends Model>> { return this.name } - getLocalUrl (this: MAccountActor | MChannelActor) { - return WEBSERVER.URL + `/accounts/` + this.Actor.preferredUsername + // Avoid error when running this method on MAccount... | MChannel... + getClientUrl (this: MAccountHost | MChannelHost) { + return WEBSERVER.URL + '/a/' + this.Actor.getIdentifier() } isBlocked () { diff --git a/server/models/actor/actor.ts b/server/models/actor/actor.ts index 80a646c77..dccb47a10 100644 --- a/server/models/actor/actor.ts +++ b/server/models/actor/actor.ts @@ -46,8 +46,8 @@ import { MActorFormattable, MActorFull, MActorHost, + MActorHostOnly, MActorId, - MActorServer, MActorSummaryFormattable, MActorUrl, MActorWithInboxes @@ -663,15 +663,15 @@ export class ActorModel extends Model>> { return this.serverId === null } - getWebfingerUrl (this: MActorServer) { + getWebfingerUrl (this: MActorHost) { return 'acct:' + this.preferredUsername + '@' + this.getHost() } - getIdentifier () { + getIdentifier (this: MActorHost) { return this.Server ? `${this.preferredUsername}@${this.Server.host}` : this.preferredUsername } - getHost (this: MActorHost) { + getHost (this: MActorHostOnly) { return this.Server ? this.Server.host : WEBSERVER.HOST } diff --git a/server/models/user/user.ts b/server/models/user/user.ts index 735b5c171..4f6a8fce4 100644 --- a/server/models/user/user.ts +++ b/server/models/user/user.ts @@ -404,6 +404,11 @@ export class UserModel extends Model>> { @Column lastLoginDate: Date + @AllowNull(false) + @Default(false) + @Column + emailPublic: boolean + @AllowNull(true) @Default(null) @Column @@ -880,6 +885,7 @@ export class UserModel extends Model>> { theme: getThemeOrDefault(this.theme, DEFAULT_USER_THEME_NAME), pendingEmail: this.pendingEmail, + emailPublic: this.emailPublic, emailVerified: this.emailVerified, nsfwPolicy: this.nsfwPolicy, diff --git a/server/models/video/formatter/video-format-utils.ts b/server/models/video/formatter/video-format-utils.ts index 6f05dbdc8..f2001e432 100644 --- a/server/models/video/formatter/video-format-utils.ts +++ b/server/models/video/formatter/video-format-utils.ts @@ -459,7 +459,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject { icon: icons.map(i => ({ type: 'Image', - url: i.getFileUrl(video), + url: i.getOriginFileUrl(video), mediaType: 'image/jpeg', width: i.width, height: i.height diff --git a/server/models/video/thumbnail.ts b/server/models/video/thumbnail.ts index f33bd3179..a4ac581e5 100644 --- a/server/models/video/thumbnail.ts +++ b/server/models/video/thumbnail.ts @@ -164,7 +164,7 @@ export class ThumbnailModel extends Model return join(directory, filename) } - getFileUrl (video: MVideo) { + getOriginFileUrl (video: MVideo) { const staticPath = ThumbnailModel.types[this.type].staticPath + this.filename if (video.isOwned()) return WEBSERVER.URL + staticPath @@ -172,6 +172,10 @@ export class ThumbnailModel extends Model return this.fileUrl } + getLocalStaticPath () { + return ThumbnailModel.types[this.type].staticPath + this.filename + } + getPath () { return ThumbnailModel.buildPath(this.type, this.filename) } diff --git a/server/models/video/video-caption.ts b/server/models/video/video-caption.ts index 2eaa77407..1fb1cae82 100644 --- a/server/models/video/video-caption.ts +++ b/server/models/video/video-caption.ts @@ -1,6 +1,6 @@ import { remove } from 'fs-extra' import { join } from 'path' -import { OrderItem, Transaction } from 'sequelize' +import { Op, OrderItem, Transaction } from 'sequelize' import { AllowNull, BeforeDestroy, @@ -166,6 +166,31 @@ export class VideoCaptionModel extends Model(query) + const result: { [ id: number ]: MVideoCaptionVideo[] } = {} + + for (const id of videoIds) { + result[id] = [] + } + + for (const caption of captions) { + result[caption.videoId].push(caption) + } + + return result + } + static getLanguageLabel (language: string) { return VIDEO_LANGUAGES[language] || 'Unknown' } diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts index 0fb52827e..19dd681a7 100644 --- a/server/models/video/video-channel.ts +++ b/server/models/video/video-channel.ts @@ -1,5 +1,8 @@ import { FindOptions, Includeable, literal, Op, QueryTypes, ScopeOptions, Transaction, WhereOptions } from 'sequelize' import { + AfterCreate, + AfterDestroy, + AfterUpdate, AllowNull, BeforeDestroy, BelongsTo, @@ -18,7 +21,8 @@ import { UpdatedAt } from 'sequelize-typescript' import { CONFIG } from '@server/initializers/config' -import { MAccountActor } from '@server/types/models' +import { InternalEventEmitter } from '@server/lib/internal-event-emitter' +import { MAccountHost } from '@server/types/models' import { forceNumber, pick } from '@shared/core-utils' import { AttributesOnly } from '@shared/typescript-utils' import { ActivityPubActor } from '../../../shared/models/activitypub' @@ -36,6 +40,7 @@ import { MChannelAP, MChannelBannerAccountDefault, MChannelFormattable, + MChannelHost, MChannelSummaryFormattable } from '../../types/models/video' import { AccountModel, ScopeNames as AccountModelScopeNames, SummaryOptions as AccountSummaryOptions } from '../account/account' @@ -416,6 +421,21 @@ export class VideoChannelModel extends Model>> { }) VideoJobInfo: VideoJobInfoModel + @AfterCreate + static notifyCreate (video: MVideo) { + InternalEventEmitter.Instance.emit('video-created', { video }) + } + + @AfterUpdate + static notifyUpdate (video: MVideo) { + InternalEventEmitter.Instance.emit('video-updated', { video }) + } + + @AfterDestroy + static notifyDestroy (video: MVideo) { + InternalEventEmitter.Instance.emit('video-deleted', { video }) + } + @BeforeDestroy - static async sendDelete (instance: MVideoAccountLight, options) { + static async sendDelete (instance: MVideoAccountLight, options: { transaction: Transaction }) { if (!instance.isOwned()) return undefined // Lazy load channels @@ -1686,15 +1704,14 @@ export class VideoModel extends Model>> { const thumbnail = this.getMiniature() if (!thumbnail) return null - return join(STATIC_PATHS.THUMBNAILS, thumbnail.filename) + return thumbnail.getLocalStaticPath() } getPreviewStaticPath () { const preview = this.getPreview() if (!preview) return null - // We use a local cache, so specify our cache endpoint instead of potential remote URL - return join(LAZY_STATIC_PATHS.PREVIEWS, preview.filename) + return preview.getLocalStaticPath() } toFormattedJSON (this: MVideoFormattable, options?: VideoFormattingJSONOptions): Video { @@ -1705,17 +1722,29 @@ export class VideoModel extends Model>> { return videoModelToFormattedDetailsJSON(this) } - getFormattedVideoFilesJSON (includeMagnet = true): VideoFile[] { + getFormattedWebVideoFilesJSON (includeMagnet = true): VideoFile[] { + return videoFilesModelToFormattedJSON(this, this.VideoFiles, { includeMagnet }) + } + + getFormattedHLSVideoFilesJSON (includeMagnet = true): VideoFile[] { + let acc: VideoFile[] = [] + + for (const p of this.VideoStreamingPlaylists) { + acc = acc.concat(videoFilesModelToFormattedJSON(this, p.VideoFiles, { includeMagnet })) + } + + return acc + } + + getFormattedAllVideoFilesJSON (includeMagnet = true): VideoFile[] { let files: VideoFile[] = [] if (Array.isArray(this.VideoFiles)) { - const result = videoFilesModelToFormattedJSON(this, this.VideoFiles, { includeMagnet }) - files = files.concat(result) + files = files.concat(this.getFormattedWebVideoFilesJSON(includeMagnet)) } - for (const p of (this.VideoStreamingPlaylists || [])) { - const result = videoFilesModelToFormattedJSON(this, p.VideoFiles, { includeMagnet }) - files = files.concat(result) + if (Array.isArray(this.VideoStreamingPlaylists)) { + files = files.concat(this.getFormattedHLSVideoFilesJSON(includeMagnet)) } return files -- cgit v1.2.3