From 244e76a552ef05a5067134b1065d26dd89246d8c Mon Sep 17 00:00:00 2001 From: Rigel Kent Date: Tue, 17 Apr 2018 00:49:04 +0200 Subject: [PATCH] feature: initial syndication feeds support MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Provides rss 2.0, atom 1.0 and json 1.0 feeds for videos (instance and account-wide) on listings and video-watch views. * still lacks redis caching * still lacks lastBuildDate support * still lacks channel-wide support * still lacks semantic annotation (for licenses, NSFW warnings, etc.) * still lacks love ( ˘ ³˘) * RSS: has MRSS support for torrent lists! * RSS: includes the first torrent in an enclosure * JSON: lists all torrents in the 'attachments' object * ATOM: lacking torrent listing support Advances #23 Partial implementation for the accountId generation in the client, which will need a hotfix to add a way to get the proper account id. --- .../account-videos.component.ts | 6 + .../src/app/shared/misc/object-length.pipe.ts | 8 + client/src/app/shared/shared.module.ts | 9 + .../app/shared/video/abstract-video-list.html | 2 +- .../app/shared/video/abstract-video-list.scss | 7 + .../app/shared/video/abstract-video-list.ts | 6 + .../shared/video/video-feed.component.html | 14 ++ .../shared/video/video-feed.component.scss | 19 ++ .../app/shared/video/video-feed.component.ts | 14 ++ client/src/app/shared/video/video.service.ts | 43 +++++ .../+video-watch/video-watch.component.html | 1 + .../+video-watch/video-watch.component.scss | 5 + .../+video-watch/video-watch.component.ts | 23 ++- .../video-list/video-local.component.ts | 11 ++ .../video-recently-added.component.ts | 10 ++ .../video-list/video-search.component.ts | 5 + .../video-list/video-trending.component.ts | 10 ++ .../src/assets/images/global/syndication.svg | 58 ++++++ client/src/sass/include/_bootstrap.scss | 2 +- package.json | 1 + server.ts | 13 +- server/controllers/feeds.ts | 136 ++++++++++++++ server/controllers/index.ts | 5 +- server/helpers/custom-validators/feeds.ts | 23 +++ server/middlewares/validators/feeds.ts | 35 ++++ server/middlewares/validators/index.ts | 1 + server/models/account/account.ts | 6 +- server/models/video/video.ts | 166 ++++++++++-------- shared/models/feeds/feed-format.enum.ts | 5 + shared/models/feeds/index.ts | 1 + shared/models/index.ts | 1 + support/doc/api/openapi.yaml | 32 ++++ yarn.lock | 10 ++ 33 files changed, 606 insertions(+), 82 deletions(-) create mode 100644 client/src/app/shared/misc/object-length.pipe.ts create mode 100644 client/src/app/shared/video/video-feed.component.html create mode 100644 client/src/app/shared/video/video-feed.component.scss create mode 100644 client/src/app/shared/video/video-feed.component.ts create mode 100644 client/src/assets/images/global/syndication.svg create mode 100644 server/controllers/feeds.ts create mode 100644 server/helpers/custom-validators/feeds.ts create mode 100644 server/middlewares/validators/feeds.ts create mode 100644 shared/models/feeds/feed-format.enum.ts create mode 100644 shared/models/feeds/index.ts diff --git a/client/src/app/account/account-videos/account-videos.component.ts b/client/src/app/account/account-videos/account-videos.component.ts index 2664d59d8..b9a3bea3f 100644 --- a/client/src/app/account/account-videos/account-videos.component.ts +++ b/client/src/app/account/account-videos/account-videos.component.ts @@ -27,6 +27,8 @@ export class AccountVideosComponent extends AbstractVideoList implements OnInit, totalItems: null } + syndicationItems = {} + protected baseVideoWidth = -1 protected baseVideoHeight = 155 @@ -61,6 +63,10 @@ export class AccountVideosComponent extends AbstractVideoList implements OnInit, return this.videoService.getMyVideos(newPagination, this.sort) } + generateSyndicationList () { + throw new Error('Method not implemented.') + } + async deleteSelectedVideos () { const toDeleteVideosIds = Object.keys(this.checkedVideos) .filter(k => this.checkedVideos[k] === true) diff --git a/client/src/app/shared/misc/object-length.pipe.ts b/client/src/app/shared/misc/object-length.pipe.ts new file mode 100644 index 000000000..84d182052 --- /dev/null +++ b/client/src/app/shared/misc/object-length.pipe.ts @@ -0,0 +1,8 @@ +import { Pipe, PipeTransform } from '@angular/core' + +@Pipe({ name: 'myObjectLength' }) +export class ObjectLengthPipe implements PipeTransform { + transform (value: Object) { + return Object.keys(value).length + } +} diff --git a/client/src/app/shared/shared.module.ts b/client/src/app/shared/shared.module.ts index eb50d45a9..74730e2aa 100644 --- a/client/src/app/shared/shared.module.ts +++ b/client/src/app/shared/shared.module.ts @@ -10,6 +10,7 @@ import { MarkdownService } from '@app/videos/shared' import { BsDropdownModule } from 'ngx-bootstrap/dropdown' import { ModalModule } from 'ngx-bootstrap/modal' +import { PopoverModule } from 'ngx-bootstrap/popover' import { TabsModule } from 'ngx-bootstrap/tabs' import { TooltipModule } from 'ngx-bootstrap/tooltip' import { BytesPipe, KeysPipe, NgPipesModule } from 'ngx-pipes' @@ -21,11 +22,13 @@ import { EditButtonComponent } from './misc/edit-button.component' import { FromNowPipe } from './misc/from-now.pipe' import { LoaderComponent } from './misc/loader.component' import { NumberFormatterPipe } from './misc/number-formatter.pipe' +import { ObjectLengthPipe } from './misc/object-length.pipe' import { RestExtractor, RestService } from './rest' import { UserService } from './users' import { VideoAbuseService } from './video-abuse' import { VideoBlacklistService } from './video-blacklist' import { VideoMiniatureComponent } from './video/video-miniature.component' +import { VideoFeedComponent } from './video/video-feed.component' import { VideoThumbnailComponent } from './video/video-thumbnail.component' import { VideoService } from './video/video.service' @@ -39,6 +42,7 @@ import { VideoService } from './video/video.service' BsDropdownModule.forRoot(), ModalModule.forRoot(), + PopoverModule.forRoot(), TabsModule.forRoot(), TooltipModule.forRoot(), @@ -50,9 +54,11 @@ import { VideoService } from './video/video.service' LoaderComponent, VideoThumbnailComponent, VideoMiniatureComponent, + VideoFeedComponent, DeleteButtonComponent, EditButtonComponent, NumberFormatterPipe, + ObjectLengthPipe, FromNowPipe, MarkdownTextareaComponent, InfiniteScrollerDirective, @@ -68,6 +74,7 @@ import { VideoService } from './video/video.service' BsDropdownModule, ModalModule, + PopoverModule, TabsModule, TooltipModule, PrimeSharedModule, @@ -77,6 +84,7 @@ import { VideoService } from './video/video.service' LoaderComponent, VideoThumbnailComponent, VideoMiniatureComponent, + VideoFeedComponent, DeleteButtonComponent, EditButtonComponent, MarkdownTextareaComponent, @@ -84,6 +92,7 @@ import { VideoService } from './video/video.service' HelpComponent, NumberFormatterPipe, + ObjectLengthPipe, FromNowPipe ], diff --git a/client/src/app/shared/video/abstract-video-list.html b/client/src/app/shared/video/abstract-video-list.html index 94a38019d..cb04e07b4 100644 --- a/client/src/app/shared/video/abstract-video-list.html +++ b/client/src/app/shared/video/abstract-video-list.html @@ -2,9 +2,9 @@
{{ titlePage }}
+
No results.
-
+ abstract generateSyndicationList () get user () { return this.authService.getUser() diff --git a/client/src/app/shared/video/video-feed.component.html b/client/src/app/shared/video/video-feed.component.html new file mode 100644 index 000000000..7733ef221 --- /dev/null +++ b/client/src/app/shared/video/video-feed.component.html @@ -0,0 +1,14 @@ +
+ + + + +
+ {{ key }} +
+ +
+ \ No newline at end of file diff --git a/client/src/app/shared/video/video-feed.component.scss b/client/src/app/shared/video/video-feed.component.scss new file mode 100644 index 000000000..2efeb405e --- /dev/null +++ b/client/src/app/shared/video/video-feed.component.scss @@ -0,0 +1,19 @@ +@import '_mixins'; + +.video-feed { + a { + @include disable-default-a-behaviour; + + color: black; + } + + .icon { + @include icon(12px); + + &.icon-syndication { + position: relative; + top: -2px; + background-image: url('../../../assets/images/global/syndication.svg'); + } + } +} \ No newline at end of file diff --git a/client/src/app/shared/video/video-feed.component.ts b/client/src/app/shared/video/video-feed.component.ts new file mode 100644 index 000000000..41257ca99 --- /dev/null +++ b/client/src/app/shared/video/video-feed.component.ts @@ -0,0 +1,14 @@ +import { Component, Input, OnChanges, SimpleChanges } from '@angular/core' + +@Component({ + selector: 'my-video-feed', + styleUrls: [ './video-feed.component.scss' ], + templateUrl: './video-feed.component.html' +}) +export class VideoFeedComponent implements OnChanges { + @Input() syndicationItems + + ngOnChanges (changes: SimpleChanges) { + this.syndicationItems = changes.syndicationItems.currentValue + } +} diff --git a/client/src/app/shared/video/video.service.ts b/client/src/app/shared/video/video.service.ts index 0a8894fd9..009155410 100644 --- a/client/src/app/shared/video/video.service.ts +++ b/client/src/app/shared/video/video.service.ts @@ -8,6 +8,7 @@ import { ResultList } from '../../../../../shared/models/result-list.model' import { UserVideoRateUpdate } from '../../../../../shared/models/videos/user-video-rate-update.model' import { UserVideoRate } from '../../../../../shared/models/videos/user-video-rate.model' import { VideoFilter } from '../../../../../shared/models/videos/video-query.type' +import { FeedFormat } from '../../../../../shared/models/feeds/feed-format.enum' import { VideoRateType } from '../../../../../shared/models/videos/video-rate.type' import { VideoUpdate } from '../../../../../shared/models/videos/video-update.model' import { environment } from '../../../environments/environment' @@ -24,6 +25,7 @@ import { objectToFormData } from '@app/shared/misc/utils' @Injectable() export class VideoService { private static BASE_VIDEO_URL = environment.apiUrl + '/api/v1/videos/' + private static BASE_FEEDS_URL = environment.apiUrl + '/feeds/videos.' constructor ( private authHttp: HttpClient, @@ -115,6 +117,47 @@ export class VideoService { .catch((res) => this.restExtractor.handleError(res)) } + baseFeed () { + const feed = {} + + for (let item in FeedFormat) { + feed[FeedFormat[item]] = VideoService.BASE_FEEDS_URL + item.toLowerCase() + } + + return feed + } + + getFeed ( + filter?: VideoFilter + ) { + let params = this.restService.addRestGetParams(new HttpParams()) + const feed = this.baseFeed() + + if (filter) { + params = params.set('filter', filter) + } + for (let item in feed) { + feed[item] = feed[item] + ((params.toString().length === 0) ? '' : '?') + params.toString() + } + + return feed + } + + getAccountFeed ( + accountId: number, + host?: string + ) { + let params = this.restService.addRestGetParams(new HttpParams()) + const feed = this.baseFeed() + + params = params.set('accountId', accountId.toString()) + for (let item in feed) { + feed[item] = feed[item] + ((params.toString().length === 0) ? '' : '?') + params.toString() + } + + return feed + } + searchVideos ( search: string, videoPagination: ComponentPagination, diff --git a/client/src/app/videos/+video-watch/video-watch.component.html b/client/src/app/videos/+video-watch/video-watch.component.html index 03f64bd12..52e3e429a 100644 --- a/client/src/app/videos/+video-watch/video-watch.component.html +++ b/client/src/app/videos/+video-watch/video-watch.component.html @@ -24,6 +24,7 @@
By {{ video.by }} Account avatar +
diff --git a/client/src/app/videos/+video-watch/video-watch.component.scss b/client/src/app/videos/+video-watch/video-watch.component.scss index 03f960339..8a3e2584b 100644 --- a/client/src/app/videos/+video-watch/video-watch.component.scss +++ b/client/src/app/videos/+video-watch/video-watch.component.scss @@ -80,6 +80,11 @@ } } + my-video-feed { + margin-left: 5px; + margin-top: 1px; + } + .video-actions-rates { display: flex; flex-direction: column; diff --git a/client/src/app/videos/+video-watch/video-watch.component.ts b/client/src/app/videos/+video-watch/video-watch.component.ts index df5b8d02d..b3ebe3e4b 100644 --- a/client/src/app/videos/+video-watch/video-watch.component.ts +++ b/client/src/app/videos/+video-watch/video-watch.component.ts @@ -1,4 +1,4 @@ -import { Component, ElementRef, NgZone, OnDestroy, OnInit, ViewChild } from '@angular/core' +import { Component, ElementRef, NgZone, OnDestroy, OnInit, ViewChild, OnChanges } from '@angular/core' import { ActivatedRoute, Router } from '@angular/router' import { RedirectService } from '@app/core/routing/redirect.service' import { peertubeLocalStorage } from '@app/shared/misc/peertube-local-storage' @@ -9,18 +9,20 @@ import { Subscription } from 'rxjs/Subscription' import * as videojs from 'video.js' import 'videojs-hotkeys' import * as WebTorrent from 'webtorrent' -import { UserVideoRateType, VideoRateType } from '../../../../../shared' +import { UserVideoRateType, VideoRateType, FeedFormat } from '../../../../../shared' import '../../../assets/player/peertube-videojs-plugin' import { AuthService, ConfirmService } from '../../core' import { VideoBlacklistService } from '../../shared' import { Account } from '../../shared/account/account.model' import { VideoDetails } from '../../shared/video/video-details.model' +import { VideoFeedComponent } from '../../shared/video/video-feed.component' import { Video } from '../../shared/video/video.model' import { VideoService } from '../../shared/video/video.service' import { MarkdownService } from '../shared' import { VideoDownloadComponent } from './modal/video-download.component' import { VideoReportComponent } from './modal/video-report.component' import { VideoShareComponent } from './modal/video-share.component' +import { environment } from '../../../environments/environment' import { getVideojsOptions } from '../../../assets/player/peertube-player' @Component({ @@ -38,6 +40,8 @@ export class VideoWatchComponent implements OnInit, OnDestroy { otherVideosDisplayed: Video[] = [] + syndicationItems = {} + player: videojs.Player playerElement: HTMLVideoElement userRating: UserVideoRateType = null @@ -98,14 +102,15 @@ export class VideoWatchComponent implements OnInit, OnDestroy { } const uuid = routeParams['uuid'] - // Video did not changed + // Video did not change if (this.video && this.video.uuid === uuid) return - + // Video did change this.videoService.getVideo(uuid).subscribe( video => { const startTime = this.route.snapshot.queryParams.start this.onVideoFetched(video, startTime) .catch(err => this.handleError(err)) + this.generateSyndicationList() }, error => { @@ -242,6 +247,16 @@ export class VideoWatchComponent implements OnInit, OnDestroy { return this.video.tags.join(', ') } + generateSyndicationList () { + const feeds = this.videoService.getAccountFeed( + this.video.account.id, + (this.video.isLocal) ? environment.apiUrl : this.video.account.host + ) + this.syndicationItems['rss 2.0'] = feeds[FeedFormat.RSS] + this.syndicationItems['atom 1.0'] = feeds[FeedFormat.ATOM] + this.syndicationItems['json 1.0'] = feeds[FeedFormat.JSON] + } + isVideoRemovable () { return this.video.isRemovableBy(this.authService.getUser()) } diff --git a/client/src/app/videos/video-list/video-local.component.ts b/client/src/app/videos/video-list/video-local.component.ts index 8f9d50a7b..9d626abd1 100644 --- a/client/src/app/videos/video-list/video-local.component.ts +++ b/client/src/app/videos/video-list/video-local.component.ts @@ -3,9 +3,12 @@ import { ActivatedRoute, Router } from '@angular/router' import { immutableAssign } from '@app/shared/misc/utils' import { NotificationsService } from 'angular2-notifications' import { AuthService } from '../../core/auth' +import { PopoverModule } from 'ngx-bootstrap/popover' import { AbstractVideoList } from '../../shared/video/abstract-video-list' import { SortField } from '../../shared/video/sort-field.type' import { VideoService } from '../../shared/video/video.service' +import { FeedFormat } from '../../../../../shared/models/feeds/feed-format.enum' +import * as url from 'url' @Component({ selector: 'my-videos-local', @@ -27,6 +30,7 @@ export class VideoLocalComponent extends AbstractVideoList implements OnInit, On ngOnInit () { super.ngOnInit() + this.generateSyndicationList() } ngOnDestroy () { @@ -38,4 +42,11 @@ export class VideoLocalComponent extends AbstractVideoList implements OnInit, On return this.videoService.getVideos(newPagination, this.sort, 'local') } + + generateSyndicationList () { + const feeds = this.videoService.getFeed('local') + this.syndicationItems['rss 2.0'] = feeds[FeedFormat.RSS] + this.syndicationItems['atom 1.0'] = feeds[FeedFormat.ATOM] + this.syndicationItems['json 1.0'] = feeds[FeedFormat.JSON] + } } diff --git a/client/src/app/videos/video-list/video-recently-added.component.ts b/client/src/app/videos/video-list/video-recently-added.component.ts index 1cecd14a0..2bdc20d92 100644 --- a/client/src/app/videos/video-list/video-recently-added.component.ts +++ b/client/src/app/videos/video-list/video-recently-added.component.ts @@ -6,6 +6,8 @@ import { AuthService } from '../../core/auth' import { AbstractVideoList } from '../../shared/video/abstract-video-list' import { SortField } from '../../shared/video/sort-field.type' import { VideoService } from '../../shared/video/video.service' +import { FeedFormat } from '../../../../../shared/models/feeds/feed-format.enum' +import * as url from 'url' @Component({ selector: 'my-videos-recently-added', @@ -27,6 +29,7 @@ export class VideoRecentlyAddedComponent extends AbstractVideoList implements On ngOnInit () { super.ngOnInit() + this.generateSyndicationList() } ngOnDestroy () { @@ -38,4 +41,11 @@ export class VideoRecentlyAddedComponent extends AbstractVideoList implements On return this.videoService.getVideos(newPagination, this.sort) } + + generateSyndicationList () { + const feeds = this.videoService.getFeed('local') + this.syndicationItems['rss 2.0'] = feeds[FeedFormat.RSS] + this.syndicationItems['atom 1.0'] = feeds[FeedFormat.ATOM] + this.syndicationItems['json 1.0'] = feeds[FeedFormat.JSON] + } } diff --git a/client/src/app/videos/video-list/video-search.component.ts b/client/src/app/videos/video-list/video-search.component.ts index b94be8e11..ef9afa757 100644 --- a/client/src/app/videos/video-list/video-search.component.ts +++ b/client/src/app/videos/video-list/video-search.component.ts @@ -7,6 +7,7 @@ import { Subscription } from 'rxjs/Subscription' import { AuthService } from '../../core/auth' import { AbstractVideoList } from '../../shared/video/abstract-video-list' import { VideoService } from '../../shared/video/video.service' +import { FeedFormat } from '../../../../../shared/models/feeds/feed-format.enum' @Component({ selector: 'my-videos-search', @@ -61,4 +62,8 @@ export class VideoSearchComponent extends AbstractVideoList implements OnInit, O const newPagination = immutableAssign(this.pagination, { currentPage: page }) return this.videoService.searchVideos(this.otherRouteParams.search, newPagination, this.sort) } + + generateSyndicationList () { + throw new Error('Method not implemented.') + } } diff --git a/client/src/app/videos/video-list/video-trending.component.ts b/client/src/app/videos/video-list/video-trending.component.ts index 1dd1ad23b..905c75ab0 100644 --- a/client/src/app/videos/video-list/video-trending.component.ts +++ b/client/src/app/videos/video-list/video-trending.component.ts @@ -6,6 +6,8 @@ import { AuthService } from '../../core/auth' import { AbstractVideoList } from '../../shared/video/abstract-video-list' import { SortField } from '../../shared/video/sort-field.type' import { VideoService } from '../../shared/video/video.service' +import { FeedFormat } from '../../../../../shared/models/feeds/feed-format.enum' +import * as url from 'url' @Component({ selector: 'my-videos-trending', @@ -27,6 +29,7 @@ export class VideoTrendingComponent extends AbstractVideoList implements OnInit, ngOnInit () { super.ngOnInit() + this.generateSyndicationList() } ngOnDestroy () { @@ -37,4 +40,11 @@ export class VideoTrendingComponent extends AbstractVideoList implements OnInit, const newPagination = immutableAssign(this.pagination, { currentPage: page }) return this.videoService.getVideos(newPagination, this.sort) } + + generateSyndicationList () { + const feeds = this.videoService.getFeed('local') + this.syndicationItems['rss 2.0'] = feeds[FeedFormat.RSS] + this.syndicationItems['atom 1.0'] = feeds[FeedFormat.ATOM] + this.syndicationItems['json 1.0'] = feeds[FeedFormat.JSON] + } } diff --git a/client/src/assets/images/global/syndication.svg b/client/src/assets/images/global/syndication.svg new file mode 100644 index 000000000..cb74cf81b --- /dev/null +++ b/client/src/assets/images/global/syndication.svg @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/client/src/sass/include/_bootstrap.scss b/client/src/sass/include/_bootstrap.scss index bbf0fda22..f15b8966e 100644 --- a/client/src/sass/include/_bootstrap.scss +++ b/client/src/sass/include/_bootstrap.scss @@ -42,7 +42,7 @@ // Components w/ JavaScript @import "~bootstrap-sass/assets/stylesheets/bootstrap/modals"; @import "~bootstrap-sass/assets/stylesheets/bootstrap/tooltip"; -//@import "~bootstrap-sass/assets/stylesheets/bootstrap/popovers"; +@import "~bootstrap-sass/assets/stylesheets/bootstrap/popovers"; //@import "~bootstrap-sass/assets/stylesheets/bootstrap/carousel"; //// Utility classes diff --git a/package.json b/package.json index 6941b913b..e3007bea9 100644 --- a/package.json +++ b/package.json @@ -81,6 +81,7 @@ "parse-torrent": "^5.8.0", "password-generator": "^2.0.2", "pem": "^1.12.3", + "pfeed": "^1.1.5", "pg": "^7.4.1", "pg-hstore": "^2.3.2", "redis": "^2.8.0", diff --git a/server.ts b/server.ts index 97941c958..06d575c86 100644 --- a/server.ts +++ b/server.ts @@ -69,7 +69,15 @@ import { installApplication } from './server/initializers' import { Emailer } from './server/lib/emailer' import { JobQueue } from './server/lib/job-queue' import { VideosPreviewCache } from './server/lib/cache' -import { apiRouter, clientsRouter, staticRouter, servicesRouter, webfingerRouter, activityPubRouter } from './server/controllers' +import { + activityPubRouter, + apiRouter, + clientsRouter, + feedsRouter, + staticRouter, + servicesRouter, + webfingerRouter +} from './server/controllers' import { Redis } from './server/lib/redis' import { BadActorFollowScheduler } from './server/lib/schedulers/bad-actor-follow-scheduler' import { RemoveOldJobsScheduler } from './server/lib/schedulers/remove-old-jobs-scheduler' @@ -144,8 +152,9 @@ app.use(apiRoute, apiRouter) // Services (oembed...) app.use('/services', servicesRouter) -app.use('/', webfingerRouter) app.use('/', activityPubRouter) +app.use('/', feedsRouter) +app.use('/', webfingerRouter) // Client files app.use('/', clientsRouter) diff --git a/server/controllers/feeds.ts b/server/controllers/feeds.ts new file mode 100644 index 000000000..b9d4c5d50 --- /dev/null +++ b/server/controllers/feeds.ts @@ -0,0 +1,136 @@ +import * as express from 'express' +import { CONFIG } from '../initializers' +import { asyncMiddleware, feedsValidator } from '../middlewares' +import { VideoModel } from '../models/video/video' +import * as Feed from 'pfeed' +import { ResultList } from '../../shared/models' +import { AccountModel } from '../models/account/account' + +const feedsRouter = express.Router() + +feedsRouter.get('/feeds/videos.:format', + asyncMiddleware(feedsValidator), + asyncMiddleware(generateFeed) +) + +// --------------------------------------------------------------------------- + +export { + feedsRouter +} + +// --------------------------------------------------------------------------- + +async function generateFeed (req: express.Request, res: express.Response, next: express.NextFunction) { + let feed = initFeed() + let feedStart = 0 + let feedCount = 10 + let feedSort = '-createdAt' + + let resultList: ResultList + const account: AccountModel = res.locals.account + + if (account) { + resultList = await VideoModel.listUserVideosForApi( + account.id, + feedStart, + feedCount, + feedSort, + true + ) + } else { + resultList = await VideoModel.listForApi( + feedStart, + feedCount, + feedSort, + req.query.filter, + true + ) + } + + // Adding video items to the feed, one at a time + resultList.data.forEach(video => { + const formattedVideoFiles = video.getFormattedVideoFilesJSON() + const torrents = formattedVideoFiles.map(videoFile => ({ + title: video.name, + url: videoFile.torrentUrl, + size_in_bytes: videoFile.size + })) + + feed.addItem({ + title: video.name, + id: video.url, + link: video.url, + description: video.getTruncatedDescription(), + content: video.description, + author: [ + { + name: video.VideoChannel.Account.getDisplayName(), + link: video.VideoChannel.Account.Actor.url + } + ], + date: video.publishedAt, + language: video.language, + nsfw: video.nsfw, + torrent: torrents + }) + }) + + // Now the feed generation is done, let's send it! + return sendFeed(feed, req, res) +} + +function initFeed () { + const webserverUrl = CONFIG.WEBSERVER.URL + + return new Feed({ + title: CONFIG.INSTANCE.NAME, + description: CONFIG.INSTANCE.SHORT_DESCRIPTION, + // updated: TODO: somehowGetLatestUpdate, // optional, default = today + id: webserverUrl, + link: webserverUrl, + image: webserverUrl + '/client/assets/images/icons/icon-96x96.png', + favicon: webserverUrl + '/client/assets/images/favicon.png', + copyright: `All rights reserved, unless otherwise specified in the terms specified at ${webserverUrl}/about` + + ` and potential licenses granted by each content's rightholder.`, + generator: `Toraifōsu`, // ^.~ + feedLinks: { + json: `${webserverUrl}/feeds/videos.json`, + atom: `${webserverUrl}/feeds/videos.atom`, + rss: `${webserverUrl}/feeds/videos.xml` + }, + author: { + name: 'instance admin of ' + CONFIG.INSTANCE.NAME, + email: CONFIG.ADMIN.EMAIL, + link: `${webserverUrl}/about` + } + }) +} + +function sendFeed (feed, req: express.Request, res: express.Response) { + const format = req.params.format + + if (format === 'atom' || format === 'atom1') { + res.set('Content-Type', 'application/atom+xml') + return res.send(feed.atom1()).end() + } + + if (format === 'json' || format === 'json1') { + res.set('Content-Type', 'application/json') + return res.send(feed.json1()).end() + } + + if (format === 'rss' || format === 'rss2') { + res.set('Content-Type', 'application/rss+xml') + return res.send(feed.rss2()).end() + } + + // We're in the ambiguous '.xml' case and we look at the format query parameter + if (req.query.format === 'atom' || req.query.format === 'atom1') { + res.set('Content-Type', 'application/atom+xml') + return res.send(feed.atom1()).end() + } + + res.set('Content-Type', 'application/rss+xml') + return res.send(feed.rss2()).end() +} diff --git a/server/controllers/index.ts b/server/controllers/index.ts index 457d0a12e..ff7928312 100644 --- a/server/controllers/index.ts +++ b/server/controllers/index.ts @@ -1,6 +1,7 @@ export * from './activitypub' -export * from './static' +export * from './api' export * from './client' +export * from './feeds' export * from './services' -export * from './api' +export * from './static' export * from './webfinger' diff --git a/server/helpers/custom-validators/feeds.ts b/server/helpers/custom-validators/feeds.ts new file mode 100644 index 000000000..638e814f0 --- /dev/null +++ b/server/helpers/custom-validators/feeds.ts @@ -0,0 +1,23 @@ +import { exists } from './misc' + +function isValidRSSFeed (value: string) { + if (!exists(value)) return false + + const feedExtensions = [ + 'xml', + 'json', + 'json1', + 'rss', + 'rss2', + 'atom', + 'atom1' + ] + + return feedExtensions.indexOf(value) !== -1 +} + +// --------------------------------------------------------------------------- + +export { + isValidRSSFeed +} diff --git a/server/middlewares/validators/feeds.ts b/server/middlewares/validators/feeds.ts new file mode 100644 index 000000000..6a8cfce86 --- /dev/null +++ b/server/middlewares/validators/feeds.ts @@ -0,0 +1,35 @@ +import * as express from 'express' +import { param, query } from 'express-validator/check' +import { isAccountIdExist, isAccountNameValid, isLocalAccountNameExist } from '../../helpers/custom-validators/accounts' +import { join } from 'path' +import { isIdOrUUIDValid } from '../../helpers/custom-validators/misc' +import { logger } from '../../helpers/logger' +import { areValidationErrors } from './utils' +import { isValidRSSFeed } from '../../helpers/custom-validators/feeds' + +const feedsValidator = [ + param('format').optional().custom(isValidRSSFeed).withMessage('Should have a valid format (rss, atom, json)'), + query('format').optional().custom(isValidRSSFeed).withMessage('Should have a valid format (rss, atom, json)'), + query('accountId').optional().custom(isIdOrUUIDValid), + query('accountName').optional().custom(isAccountNameValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking feeds parameters', { parameters: req.query }) + + if (areValidationErrors(req, res)) return + + if (req.query.accountId) { + if (!await isAccountIdExist(req.query.accountId, res)) return + } else if (req.query.accountName) { + if (!await isLocalAccountNameExist(req.query.accountName, res)) return + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + feedsValidator +} diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts index 9840e8f65..b69e1f14b 100644 --- a/server/middlewares/validators/index.ts +++ b/server/middlewares/validators/index.ts @@ -3,6 +3,7 @@ export * from './oembed' export * from './activitypub' export * from './pagination' export * from './follows' +export * from './feeds' export * from './sort' export * from './users' export * from './videos' diff --git a/server/models/account/account.ts b/server/models/account/account.ts index c5955ef3b..3ff59887d 100644 --- a/server/models/account/account.ts +++ b/server/models/account/account.ts @@ -246,7 +246,7 @@ export class AccountModel extends Model { const actor = this.Actor.toFormattedJSON() const account = { id: this.id, - displayName: this.name, + displayName: this.getDisplayName(), description: this.description, createdAt: this.createdAt, updatedAt: this.updatedAt @@ -266,4 +266,8 @@ export class AccountModel extends Model { isOwned () { return this.Actor.isOwned() } + + getDisplayName () { + return this.name + } } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 54fe54535..240a2b5a2 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -95,14 +95,15 @@ enum ScopeNames { } @Scopes({ - [ScopeNames.AVAILABLE_FOR_LIST]: (actorId: number, filter?: VideoFilter) => ({ - where: { - id: { - [Sequelize.Op.notIn]: Sequelize.literal( - '(SELECT "videoBlacklist"."videoId" FROM "videoBlacklist")' - ), - [ Sequelize.Op.in ]: Sequelize.literal( - '(' + + [ScopeNames.AVAILABLE_FOR_LIST]: (actorId: number, filter?: VideoFilter, withFiles?: boolean) => { + const query: IFindOptions = { + where: { + id: { + [Sequelize.Op.notIn]: Sequelize.literal( + '(SELECT "videoBlacklist"."videoId" FROM "videoBlacklist")' + ), + [ Sequelize.Op.in ]: Sequelize.literal( + '(' + 'SELECT "videoShare"."videoId" AS "id" FROM "videoShare" ' + 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "videoShare"."actorId" ' + 'WHERE "actorFollow"."actorId" = ' + parseInt(actorId.toString(), 10) + @@ -113,45 +114,55 @@ enum ScopeNames { 'INNER JOIN "actor" ON "account"."actorId" = "actor"."id" ' + 'LEFT JOIN "actorFollow" ON "actorFollow"."targetActorId" = "actor"."id" ' + 'WHERE "actor"."serverId" IS NULL OR "actorFollow"."actorId" = ' + parseInt(actorId.toString(), 10) + - ')' - ) + ')' + ) + }, + privacy: VideoPrivacy.PUBLIC }, - privacy: VideoPrivacy.PUBLIC - }, - include: [ - { - attributes: [ 'name', 'description' ], - model: VideoChannelModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'name' ], - model: AccountModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'preferredUsername', 'url', 'serverId' ], - model: ActorModel.unscoped(), - required: true, - where: VideoModel.buildActorWhereWithFilter(filter), - include: [ - { - attributes: [ 'host' ], - model: ServerModel.unscoped(), - required: false - }, - { - model: AvatarModel.unscoped(), - required: false - } - ] - } - ] - } - ] - } - ] - }), + include: [ + { + attributes: [ 'name', 'description' ], + model: VideoChannelModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'name' ], + model: AccountModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'preferredUsername', 'url', 'serverId', 'avatarId' ], + model: ActorModel.unscoped(), + required: true, + where: VideoModel.buildActorWhereWithFilter(filter), + include: [ + { + attributes: [ 'host' ], + model: ServerModel.unscoped(), + required: false + }, + { + model: AvatarModel.unscoped(), + required: false + } + ] + } + ] + } + ] + } + ] + } + + if (withFiles === true) { + query.include.push({ + model: VideoFileModel.unscoped(), + required: true + }) + } + + return query + }, [ScopeNames.WITH_ACCOUNT_DETAILS]: { include: [ { @@ -629,8 +640,8 @@ export class VideoModel extends Model { }) } - static listUserVideosForApi (userId: number, start: number, count: number, sort: string) { - const query = { + static listUserVideosForApi (userId: number, start: number, count: number, sort: string, withFiles = false) { + const query: IFindOptions = { offset: start, limit: count, order: getSort(sort), @@ -651,6 +662,13 @@ export class VideoModel extends Model { ] } + if (withFiles === true) { + query.include.push({ + model: VideoFileModel.unscoped(), + required: true + }) + } + return VideoModel.findAndCountAll(query).then(({ rows, count }) => { return { data: rows, @@ -659,7 +677,7 @@ export class VideoModel extends Model { }) } - static async listForApi (start: number, count: number, sort: string, filter?: VideoFilter) { + static async listForApi (start: number, count: number, sort: string, filter?: VideoFilter, withFiles = false) { const query = { offset: start, limit: count, @@ -668,7 +686,7 @@ export class VideoModel extends Model { const serverActor = await getServerActor() - return VideoModel.scope({ method: [ ScopeNames.AVAILABLE_FOR_LIST, serverActor.id, filter ] }) + return VideoModel.scope({ method: [ ScopeNames.AVAILABLE_FOR_LIST, serverActor.id, filter, withFiles ] }) .findAndCountAll(query) .then(({ rows, count }) => { return { @@ -707,7 +725,8 @@ export class VideoModel extends Model { const serverActor = await getServerActor() return VideoModel.scope({ method: [ ScopeNames.AVAILABLE_FOR_LIST, serverActor.id ] }) - .findAndCountAll(query).then(({ rows, count }) => { + .findAndCountAll(query) + .then(({ rows, count }) => { return { data: rows, total: count @@ -1006,29 +1025,34 @@ export class VideoModel extends Model { } // Format and sort video files + detailsJson.files = this.getFormattedVideoFilesJSON() + + return Object.assign(formattedJson, detailsJson) + } + + getFormattedVideoFilesJSON (): VideoFile[] { const { baseUrlHttp, baseUrlWs } = this.getBaseUrls() - detailsJson.files = this.VideoFiles - .map(videoFile => { - let resolutionLabel = videoFile.resolution + 'p' - return { - resolution: { - id: videoFile.resolution, - label: resolutionLabel - }, - magnetUri: this.generateMagnetUri(videoFile, baseUrlHttp, baseUrlWs), - size: videoFile.size, - torrentUrl: this.getTorrentUrl(videoFile, baseUrlHttp), - fileUrl: this.getVideoFileUrl(videoFile, baseUrlHttp) - } as VideoFile - }) - .sort((a, b) => { - if (a.resolution.id < b.resolution.id) return 1 - if (a.resolution.id === b.resolution.id) return 0 - return -1 - }) + return this.VideoFiles + .map(videoFile => { + let resolutionLabel = videoFile.resolution + 'p' - return Object.assign(formattedJson, detailsJson) + return { + resolution: { + id: videoFile.resolution, + label: resolutionLabel + }, + magnetUri: this.generateMagnetUri(videoFile, baseUrlHttp, baseUrlWs), + size: videoFile.size, + torrentUrl: this.getTorrentUrl(videoFile, baseUrlHttp), + fileUrl: this.getVideoFileUrl(videoFile, baseUrlHttp) + } as VideoFile + }) + .sort((a, b) => { + if (a.resolution.id < b.resolution.id) return 1 + if (a.resolution.id === b.resolution.id) return 0 + return -1 + }) } toActivityPubObject (): VideoTorrentObject { diff --git a/shared/models/feeds/feed-format.enum.ts b/shared/models/feeds/feed-format.enum.ts new file mode 100644 index 000000000..f3173a781 --- /dev/null +++ b/shared/models/feeds/feed-format.enum.ts @@ -0,0 +1,5 @@ +export enum FeedFormat { + RSS = 'xml', + ATOM = 'atom', + JSON = 'json' +} diff --git a/shared/models/feeds/index.ts b/shared/models/feeds/index.ts new file mode 100644 index 000000000..d56c8458c --- /dev/null +++ b/shared/models/feeds/index.ts @@ -0,0 +1 @@ +export * from './feed-format.enum' diff --git a/shared/models/index.ts b/shared/models/index.ts index ae3a44777..95bc402d6 100644 --- a/shared/models/index.ts +++ b/shared/models/index.ts @@ -2,6 +2,7 @@ export * from './actors' export * from './activitypub' export * from './users' export * from './videos' +export * from './feeds' export * from './server/job.model' export * from './oauth-client-local.model' export * from './result-list.model' diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yaml index c3efa512d..0729ac8ec 100644 --- a/support/doc/api/openapi.yaml +++ b/support/doc/api/openapi.yaml @@ -78,6 +78,38 @@ paths: description: successful operation schema: $ref: '#/definitions/ServerConfig' + /feeds/videos.{format}: + get: + tags: + - Feeds + consumes: + - application/json + produces: + - application/json + parameters: + - name: format + in: path + required: true + type: string + enum: ['xml', 'atom' 'json'] + default: 'xml' + description: 'The format expected (xml defaults to RSS 2.0, atom to ATOM 1.0 and json to JSON FEED 1.0' + - name: accountId + in: query + required: false + type: number + description: 'The id of the local account to filter to (beware, users IDs and not actors IDs which will return empty feeds' + - name: accountName + in: query + required: false + type: string + description: 'The name of the local account to filter to' + responses: + '200': + description: successful operation + content: + application/json: + application/xml: /jobs: get: security: diff --git a/yarn.lock b/yarn.lock index b4c3b7bcc..2b445860f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4585,6 +4585,12 @@ performance-now@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" +pfeed@^1.1.2: + version "1.1.5" + resolved "https://registry.yarnpkg.com/pfeed/-/pfeed-1.1.5.tgz#6d0ab54209c60b45de03a15efaab7be867a3f71a" + dependencies: + xml "^1.0.1" + pg-connection-string@0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" @@ -6792,6 +6798,10 @@ xhr2@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/xhr2/-/xhr2-0.1.4.tgz#7f87658847716db5026323812f818cadab387a5f" +xml@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5" + xmldom@0.1.19: version "0.1.19" resolved "https://registry.yarnpkg.com/xmldom/-/xmldom-0.1.19.tgz#631fc07776efd84118bf25171b37ed4d075a0abc" -- 2.41.0