From: Chocobozzz Date: Tue, 11 Sep 2018 14:27:07 +0000 (+0200) Subject: Basic video redundancy implementation X-Git-Tag: v1.0.0-beta.14~80 X-Git-Url: https://git.immae.eu/?a=commitdiff_plain;h=c48e82b5e0478434de30626d14594a97f2402e7c;p=github%2FChocobozzz%2FPeerTube.git Basic video redundancy implementation --- diff --git a/client/src/app/+admin/admin.module.ts b/client/src/app/+admin/admin.module.ts index e94aac1eb..5784609ef 100644 --- a/client/src/app/+admin/admin.module.ts +++ b/client/src/app/+admin/admin.module.ts @@ -14,6 +14,8 @@ import { UserCreateComponent, UserListComponent, UsersComponent, UserService, Us import { ModerationCommentModalComponent, VideoAbuseListComponent, VideoBlacklistListComponent } from './moderation' import { UserBanModalComponent } from '@app/+admin/users/user-list/user-ban-modal.component' import { ModerationComponent } from '@app/+admin/moderation/moderation.component' +import { RedundancyCheckboxComponent } from '@app/+admin/follows/shared/redundancy-checkbox.component' +import { RedundancyService } from '@app/+admin/follows/shared/redundancy.service' @NgModule({ imports: [ @@ -29,6 +31,7 @@ import { ModerationComponent } from '@app/+admin/moderation/moderation.component FollowingAddComponent, FollowersListComponent, FollowingListComponent, + RedundancyCheckboxComponent, UsersComponent, UserCreateComponent, @@ -54,6 +57,7 @@ import { ModerationComponent } from '@app/+admin/moderation/moderation.component providers: [ FollowService, + RedundancyService, UserService, JobService, ConfigService diff --git a/client/src/app/+admin/follows/followers-list/followers-list.component.ts b/client/src/app/+admin/follows/followers-list/followers-list.component.ts index 96fb67588..ca993dcd3 100644 --- a/client/src/app/+admin/follows/followers-list/followers-list.component.ts +++ b/client/src/app/+admin/follows/followers-list/followers-list.component.ts @@ -2,7 +2,7 @@ import { Component, OnInit } from '@angular/core' import { NotificationsService } from 'angular2-notifications' import { SortMeta } from 'primeng/primeng' -import { AccountFollow } from '../../../../../../shared/models/actors/follow.model' +import { ActorFollow } from '../../../../../../shared/models/actors/follow.model' import { RestPagination, RestTable } from '../../../shared' import { FollowService } from '../shared' import { I18n } from '@ngx-translate/i18n-polyfill' @@ -13,7 +13,7 @@ import { I18n } from '@ngx-translate/i18n-polyfill' styleUrls: [ './followers-list.component.scss' ] }) export class FollowersListComponent extends RestTable implements OnInit { - followers: AccountFollow[] = [] + followers: ActorFollow[] = [] totalRecords = 0 rowsPerPage = 10 sort: SortMeta = { field: 'createdAt', order: 1 } diff --git a/client/src/app/+admin/follows/following-list/following-list.component.html b/client/src/app/+admin/follows/following-list/following-list.component.html index e4a45e88c..66ab64c50 100644 --- a/client/src/app/+admin/follows/following-list/following-list.component.html +++ b/client/src/app/+admin/follows/following-list/following-list.component.html @@ -8,6 +8,7 @@ Host State Created + Redundancy allowed @@ -18,6 +19,11 @@ {{ follow.following.host }} {{ follow.state }} {{ follow.createdAt }} + + + diff --git a/client/src/app/+admin/follows/following-list/following-list.component.scss b/client/src/app/+admin/follows/following-list/following-list.component.scss new file mode 100644 index 000000000..bfcdcaa49 --- /dev/null +++ b/client/src/app/+admin/follows/following-list/following-list.component.scss @@ -0,0 +1,13 @@ +@import '_variables'; +@import '_mixins'; + +my-redundancy-checkbox /deep/ my-peertube-checkbox { + .form-group { + margin-bottom: 0; + align-items: center; + } + + label { + margin: 0; + } +} \ No newline at end of file diff --git a/client/src/app/+admin/follows/following-list/following-list.component.ts b/client/src/app/+admin/follows/following-list/following-list.component.ts index 06e341e68..dd57884c6 100644 --- a/client/src/app/+admin/follows/following-list/following-list.component.ts +++ b/client/src/app/+admin/follows/following-list/following-list.component.ts @@ -1,7 +1,7 @@ import { Component, OnInit } from '@angular/core' import { NotificationsService } from 'angular2-notifications' import { SortMeta } from 'primeng/primeng' -import { AccountFollow } from '../../../../../../shared/models/actors/follow.model' +import { ActorFollow } from '../../../../../../shared/models/actors/follow.model' import { ConfirmService } from '../../../core/confirm/confirm.service' import { RestPagination, RestTable } from '../../../shared' import { FollowService } from '../shared' @@ -9,10 +9,11 @@ import { I18n } from '@ngx-translate/i18n-polyfill' @Component({ selector: 'my-followers-list', - templateUrl: './following-list.component.html' + templateUrl: './following-list.component.html', + styleUrls: [ './following-list.component.scss' ] }) export class FollowingListComponent extends RestTable implements OnInit { - following: AccountFollow[] = [] + following: ActorFollow[] = [] totalRecords = 0 rowsPerPage = 10 sort: SortMeta = { field: 'createdAt', order: 1 } @@ -31,7 +32,7 @@ export class FollowingListComponent extends RestTable implements OnInit { this.loadSort() } - async removeFollowing (follow: AccountFollow) { + async removeFollowing (follow: ActorFollow) { const res = await this.confirmService.confirm( this.i18n('Do you really want to unfollow {{host}}?', { host: follow.following.host }), this.i18n('Unfollow') diff --git a/client/src/app/+admin/follows/shared/follow.service.ts b/client/src/app/+admin/follows/shared/follow.service.ts index 87ea5fb0c..27169a9cd 100644 --- a/client/src/app/+admin/follows/shared/follow.service.ts +++ b/client/src/app/+admin/follows/shared/follow.service.ts @@ -3,7 +3,7 @@ import { HttpClient, HttpParams } from '@angular/common/http' import { Injectable } from '@angular/core' import { SortMeta } from 'primeng/primeng' import { Observable } from 'rxjs' -import { AccountFollow, ResultList } from '../../../../../../shared' +import { ActorFollow, ResultList } from '../../../../../../shared' import { environment } from '../../../../environments/environment' import { RestExtractor, RestPagination, RestService } from '../../../shared' @@ -18,22 +18,22 @@ export class FollowService { ) { } - getFollowing (pagination: RestPagination, sort: SortMeta): Observable> { + getFollowing (pagination: RestPagination, sort: SortMeta): Observable> { let params = new HttpParams() params = this.restService.addRestGetParams(params, pagination, sort) - return this.authHttp.get>(FollowService.BASE_APPLICATION_URL + '/following', { params }) + return this.authHttp.get>(FollowService.BASE_APPLICATION_URL + '/following', { params }) .pipe( map(res => this.restExtractor.convertResultListDateToHuman(res)), catchError(res => this.restExtractor.handleError(res)) ) } - getFollowers (pagination: RestPagination, sort: SortMeta): Observable> { + getFollowers (pagination: RestPagination, sort: SortMeta): Observable> { let params = new HttpParams() params = this.restService.addRestGetParams(params, pagination, sort) - return this.authHttp.get>(FollowService.BASE_APPLICATION_URL + '/followers', { params }) + return this.authHttp.get>(FollowService.BASE_APPLICATION_URL + '/followers', { params }) .pipe( map(res => this.restExtractor.convertResultListDateToHuman(res)), catchError(res => this.restExtractor.handleError(res)) @@ -52,7 +52,7 @@ export class FollowService { ) } - unfollow (follow: AccountFollow) { + unfollow (follow: ActorFollow) { return this.authHttp.delete(FollowService.BASE_APPLICATION_URL + '/following/' + follow.following.host) .pipe( map(this.restExtractor.extractDataBool), diff --git a/client/src/app/+admin/follows/shared/redundancy-checkbox.component.html b/client/src/app/+admin/follows/shared/redundancy-checkbox.component.html new file mode 100644 index 000000000..8a57d65f0 --- /dev/null +++ b/client/src/app/+admin/follows/shared/redundancy-checkbox.component.html @@ -0,0 +1,3 @@ + \ No newline at end of file diff --git a/client/src/app/+admin/follows/shared/redundancy-checkbox.component.scss b/client/src/app/+admin/follows/shared/redundancy-checkbox.component.scss new file mode 100644 index 000000000..5e6774739 --- /dev/null +++ b/client/src/app/+admin/follows/shared/redundancy-checkbox.component.scss @@ -0,0 +1,2 @@ +@import '_variables'; +@import '_mixins'; diff --git a/client/src/app/+admin/follows/shared/redundancy-checkbox.component.ts b/client/src/app/+admin/follows/shared/redundancy-checkbox.component.ts new file mode 100644 index 000000000..ff4725e91 --- /dev/null +++ b/client/src/app/+admin/follows/shared/redundancy-checkbox.component.ts @@ -0,0 +1,42 @@ +import { Component, Input } from '@angular/core' +import { AuthService } from '@app/core' +import { RestExtractor } from '@app/shared/rest' +import { RedirectService } from '@app/core/routing/redirect.service' +import { NotificationsService } from 'angular2-notifications' +import { I18n } from '@ngx-translate/i18n-polyfill' +import { RedundancyService } from '@app/+admin/follows/shared/redundancy.service' + +@Component({ + selector: 'my-redundancy-checkbox', + templateUrl: './redundancy-checkbox.component.html', + styleUrls: [ './redundancy-checkbox.component.scss' ] +}) +export class RedundancyCheckboxComponent { + @Input() redundancyAllowed: boolean + @Input() host: string + + constructor ( + private authService: AuthService, + private restExtractor: RestExtractor, + private redirectService: RedirectService, + private notificationsService: NotificationsService, + private redundancyService: RedundancyService, + private i18n: I18n + ) { } + + updateRedundancyState () { + this.redundancyService.updateRedundancy(this.host, this.redundancyAllowed) + .subscribe( + () => { + const stateLabel = this.redundancyAllowed ? this.i18n('enabled') : this.i18n('disabled') + + this.notificationsService.success( + this.i18n('Success'), + this.i18n('Redundancy for {{host}} is {{stateLabel}}', { host: this.host, stateLabel }) + ) + }, + + err => this.notificationsService.error(this.i18n('Error'), err.message) + ) + } +} diff --git a/client/src/app/+admin/follows/shared/redundancy.service.ts b/client/src/app/+admin/follows/shared/redundancy.service.ts new file mode 100644 index 000000000..96b29faab --- /dev/null +++ b/client/src/app/+admin/follows/shared/redundancy.service.ts @@ -0,0 +1,29 @@ +import { catchError, map } from 'rxjs/operators' +import { HttpClient } from '@angular/common/http' +import { Injectable } from '@angular/core' +import { RestExtractor, RestService } from '@app/shared' +import { environment } from '../../../../environments/environment' + +@Injectable() +export class RedundancyService { + static BASE_USER_SUBSCRIPTIONS_URL = environment.apiUrl + '/api/v1/server/redundancy' + + constructor ( + private authHttp: HttpClient, + private restExtractor: RestExtractor, + private restService: RestService + ) { } + + updateRedundancy (host: string, redundancyAllowed: boolean) { + const url = RedundancyService.BASE_USER_SUBSCRIPTIONS_URL + '/' + host + + const body = { redundancyAllowed } + + return this.authHttp.put(url, body) + .pipe( + map(this.restExtractor.extractDataBool), + catchError(err => this.restExtractor.handleError(err)) + ) + } + +} diff --git a/config/default.yaml b/config/default.yaml index 40458bb38..af29a4379 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -66,6 +66,15 @@ trending: videos: interval_days: 7 # Compute trending videos for the last x days +# Cache remote videos on your server, to help other instances to broadcast the video +# You can define multiple caches using different sizes/strategies +# Once you have defined your strategies, choose which instances you want to cache in admin -> manage follows -> following +redundancy: + videos: +# - +# size: '10GB' +# strategy: 'most-views' # Cache videos that have the most views + cache: previews: size: 500 # Max number of previews you want to cache diff --git a/config/production.yaml.example b/config/production.yaml.example index d032e4a0c..ddd43093f 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -67,6 +67,15 @@ trending: videos: interval_days: 7 # Compute trending videos for the last x days +# Cache remote videos on your server, to help other instances to broadcast the video +# You can define multiple caches using different sizes/strategies +# Once you have defined your strategies, choose which instances you want to cache in admin -> manage follows -> following +redundancy: + videos: +# - +# size: '10GB' +# strategy: 'most-views' # Cache videos that have the most views + ############################################################################### # # From this point, all the following keys can be overridden by the web interface diff --git a/config/test.yaml b/config/test.yaml index 6a8e47aac..0f280eabd 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -21,6 +21,12 @@ smtp: log: level: 'debug' +redundancy: + videos: + - + size: '100KB' + strategy: 'most-views' + cache: previews: size: 1 diff --git a/package.json b/package.json index fafe4b296..5a8843b0c 100644 --- a/package.json +++ b/package.json @@ -86,6 +86,7 @@ "bluebird": "^3.5.0", "body-parser": "^1.12.4", "bull": "^3.4.2", + "bytes": "^3.0.0", "commander": "^2.13.0", "concurrently": "^4.0.1", "config": "^2.0.1", @@ -145,6 +146,7 @@ "@types/bluebird": "3.5.21", "@types/body-parser": "^1.16.3", "@types/bull": "^3.3.12", + "@types/bytes": "^3.0.0", "@types/chai": "^4.0.4", "@types/chai-json-schema": "^1.4.3", "@types/chai-xml": "^0.3.1", diff --git a/server.ts b/server.ts index 76d00edd3..8bc5e5f32 100644 --- a/server.ts +++ b/server.ts @@ -94,6 +94,7 @@ import { BadActorFollowScheduler } from './server/lib/schedulers/bad-actor-follo import { RemoveOldJobsScheduler } from './server/lib/schedulers/remove-old-jobs-scheduler' import { UpdateVideosScheduler } from './server/lib/schedulers/update-videos-scheduler' import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler' +import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler' // ----------- Command line ----------- @@ -206,6 +207,7 @@ async function startApplication () { RemoveOldJobsScheduler.Instance.enable() UpdateVideosScheduler.Instance.enable() YoutubeDlUpdateScheduler.Instance.enable() + VideosRedundancyScheduler.Instance.enable() // Redis initialization Redis.Instance.init() diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts index 54cf44419..2e168ea78 100644 --- a/server/controllers/activitypub/client.ts +++ b/server/controllers/activitypub/client.ts @@ -3,9 +3,9 @@ import * as express from 'express' import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos' import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub' import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../initializers' -import { buildVideoAnnounce } from '../../lib/activitypub/send' +import { buildAnnounceWithVideoAudience } from '../../lib/activitypub/send' import { audiencify, getAudience } from '../../lib/activitypub/audience' -import { createActivityData } from '../../lib/activitypub/send/send-create' +import { buildCreateActivity } from '../../lib/activitypub/send/send-create' import { asyncMiddleware, executeIfActivityPub, localAccountValidator, localVideoChannelValidator } from '../../middlewares' import { videosGetValidator, videosShareValidator } from '../../middlewares/validators' import { videoCommentGetValidator } from '../../middlewares/validators/video-comments' @@ -26,6 +26,8 @@ import { getVideoSharesActivityPubUrl } from '../../lib/activitypub' import { VideoCaptionModel } from '../../models/video/video-caption' +import { videoRedundancyGetValidator } from '../../middlewares/validators/redundancy' +import { getServerActor } from '../../helpers/utils' const activityPubClientRouter = express.Router() @@ -93,6 +95,11 @@ activityPubClientRouter.get('/video-channels/:name/following', executeIfActivityPub(asyncMiddleware(videoChannelFollowingController)) ) +activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?', + executeIfActivityPub(asyncMiddleware(videoRedundancyGetValidator)), + executeIfActivityPub(asyncMiddleware(videoRedundancyController)) +) + // --------------------------------------------------------------------------- export { @@ -131,7 +138,7 @@ async function videoController (req: express.Request, res: express.Response, nex const videoObject = audiencify(video.toActivityPubObject(), audience) if (req.path.endsWith('/activity')) { - const data = createActivityData(video.url, video.VideoChannel.Account.Actor, videoObject, audience) + const data = buildCreateActivity(video.url, video.VideoChannel.Account.Actor, videoObject, audience) return activityPubResponse(activityPubContextify(data), res) } @@ -140,9 +147,9 @@ async function videoController (req: express.Request, res: express.Response, nex async function videoAnnounceController (req: express.Request, res: express.Response, next: express.NextFunction) { const share = res.locals.videoShare as VideoShareModel - const object = await buildVideoAnnounce(share.Actor, share, res.locals.video, undefined) + const { activity } = await buildAnnounceWithVideoAudience(share.Actor, share, res.locals.video, undefined) - return activityPubResponse(activityPubContextify(object), res) + return activityPubResponse(activityPubContextify(activity), res) } async function videoAnnouncesController (req: express.Request, res: express.Response, next: express.NextFunction) { @@ -219,13 +226,28 @@ async function videoCommentController (req: express.Request, res: express.Respon const videoCommentObject = audiencify(videoComment.toActivityPubObject(threadParentComments), audience) if (req.path.endsWith('/activity')) { - const data = createActivityData(videoComment.url, videoComment.Account.Actor, videoCommentObject, audience) + const data = buildCreateActivity(videoComment.url, videoComment.Account.Actor, videoCommentObject, audience) return activityPubResponse(activityPubContextify(data), res) } return activityPubResponse(activityPubContextify(videoCommentObject), res) } +async function videoRedundancyController (req: express.Request, res: express.Response) { + const videoRedundancy = res.locals.videoRedundancy + const serverActor = await getServerActor() + + const audience = getAudience(serverActor) + const object = audiencify(videoRedundancy.toActivityPubObject(), audience) + + if (req.path.endsWith('/activity')) { + const data = buildCreateActivity(videoRedundancy.url, serverActor, object, audience) + return activityPubResponse(activityPubContextify(data), res) + } + + return activityPubResponse(activityPubContextify(object), res) +} + // --------------------------------------------------------------------------- async function actorFollowing (req: express.Request, actor: ActorModel) { diff --git a/server/controllers/activitypub/outbox.ts b/server/controllers/activitypub/outbox.ts index db69ae54b..bd0e4fe9d 100644 --- a/server/controllers/activitypub/outbox.ts +++ b/server/controllers/activitypub/outbox.ts @@ -3,7 +3,7 @@ import { Activity } from '../../../shared/models/activitypub/activity' import { VideoPrivacy } from '../../../shared/models/videos' import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub' import { logger } from '../../helpers/logger' -import { announceActivityData, createActivityData } from '../../lib/activitypub/send' +import { buildAnnounceActivity, buildCreateActivity } from '../../lib/activitypub/send' import { buildAudience } from '../../lib/activitypub/audience' import { asyncMiddleware, localAccountValidator, localVideoChannelValidator } from '../../middlewares' import { AccountModel } from '../../models/account/account' @@ -60,12 +60,12 @@ async function buildActivities (actor: ActorModel, start: number, count: number) // This is a shared video if (video.VideoShares !== undefined && video.VideoShares.length !== 0) { const videoShare = video.VideoShares[0] - const announceActivity = announceActivityData(videoShare.url, actor, video.url, createActivityAudience) + const announceActivity = buildAnnounceActivity(videoShare.url, actor, video.url, createActivityAudience) activities.push(announceActivity) } else { const videoObject = video.toActivityPubObject() - const createActivity = createActivityData(video.url, byActor, videoObject, createActivityAudience) + const createActivity = buildCreateActivity(video.url, byActor, videoObject, createActivityAudience) activities.push(createActivity) } diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts index bb7174891..28a7a04ca 100644 --- a/server/controllers/api/search.ts +++ b/server/controllers/api/search.ts @@ -17,8 +17,6 @@ import { import { VideoChannelsSearchQuery, VideosSearchQuery } from '../../../shared/models/search' import { getOrCreateActorAndServerAndModel, getOrCreateVideoAndAccountAndChannel } from '../../lib/activitypub' import { logger } from '../../helpers/logger' -import { User } from '../../../shared/models/users' -import { CONFIG } from '../../initializers/constants' import { VideoChannelModel } from '../../models/video/video-channel' import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger' diff --git a/server/controllers/api/server/follows.ts b/server/controllers/api/server/follows.ts index 23308445f..a4eae6b45 100644 --- a/server/controllers/api/server/follows.ts +++ b/server/controllers/api/server/follows.ts @@ -96,6 +96,11 @@ async function removeFollow (req: express.Request, res: express.Response, next: await sequelizeTypescript.transaction(async t => { if (follow.state === 'accepted') await sendUndoFollow(follow, t) + // Disable redundancy on unfollowed instances + const server = follow.ActorFollowing.Server + server.redundancyAllowed = false + await server.save({ transaction: t }) + await follow.destroy({ transaction: t }) }) diff --git a/server/controllers/api/server/index.ts b/server/controllers/api/server/index.ts index 850a52cdb..43bca2c10 100644 --- a/server/controllers/api/server/index.ts +++ b/server/controllers/api/server/index.ts @@ -1,10 +1,12 @@ import * as express from 'express' import { serverFollowsRouter } from './follows' import { statsRouter } from './stats' +import { serverRedundancyRouter } from './redundancy' const serverRouter = express.Router() serverRouter.use('/', serverFollowsRouter) +serverRouter.use('/', serverRedundancyRouter) serverRouter.use('/', statsRouter) // --------------------------------------------------------------------------- diff --git a/server/controllers/api/server/redundancy.ts b/server/controllers/api/server/redundancy.ts new file mode 100644 index 000000000..4216b9e35 --- /dev/null +++ b/server/controllers/api/server/redundancy.ts @@ -0,0 +1,32 @@ +import * as express from 'express' +import { UserRight } from '../../../../shared/models/users' +import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares' +import { updateServerRedundancyValidator } from '../../../middlewares/validators/redundancy' +import { ServerModel } from '../../../models/server/server' + +const serverRedundancyRouter = express.Router() + +serverRedundancyRouter.put('/redundancy/:host', + authenticate, + ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW), + asyncMiddleware(updateServerRedundancyValidator), + asyncMiddleware(updateRedundancy) +) + +// --------------------------------------------------------------------------- + +export { + serverRedundancyRouter +} + +// --------------------------------------------------------------------------- + +async function updateRedundancy (req: express.Request, res: express.Response, next: express.NextFunction) { + const server = res.locals.server as ServerModel + + server.redundancyAllowed = req.body.redundancyAllowed + + await server.save() + + return res.sendStatus(204) +} diff --git a/server/controllers/api/videos/abuse.ts b/server/controllers/api/videos/abuse.ts index 59bdf6257..08e11b00b 100644 --- a/server/controllers/api/videos/abuse.ts +++ b/server/controllers/api/videos/abuse.ts @@ -112,7 +112,7 @@ async function reportVideoAbuse (req: express.Request, res: express.Response) { // We send the video abuse to the origin server if (videoInstance.isOwned() === false) { - await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance, t) + await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance) } auditLogger.create(reporterAccount.Actor.getIdentifier(), new VideoAbuseAuditView(videoAbuseInstance.toFormattedJSON())) diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index a9de11fb0..1304c7559 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts @@ -14,20 +14,24 @@ function activityPubContextify (data: T) { 'https://w3id.org/security/v1', { RsaSignature2017: 'https://w3id.org/security#RsaSignature2017', + pt: 'https://joinpeertube.org/ns', + schema: 'http://schema.org#', Hashtag: 'as:Hashtag', - uuid: 'http://schema.org/identifier', - category: 'http://schema.org/category', - licence: 'http://schema.org/license', - subtitleLanguage: 'http://schema.org/subtitleLanguage', + uuid: 'schema:identifier', + category: 'schema:category', + licence: 'schema:license', + subtitleLanguage: 'schema:subtitleLanguage', sensitive: 'as:sensitive', - language: 'http://schema.org/inLanguage', - views: 'http://schema.org/Number', - stats: 'http://schema.org/Number', - size: 'http://schema.org/Number', - fps: 'http://schema.org/Number', - commentsEnabled: 'http://schema.org/Boolean', - waitTranscoding: 'http://schema.org/Boolean', - support: 'http://schema.org/Text' + language: 'schema:inLanguage', + views: 'schema:Number', + stats: 'schema:Number', + size: 'schema:Number', + fps: 'schema:Number', + commentsEnabled: 'schema:Boolean', + waitTranscoding: 'schema:Boolean', + expires: 'schema:expires', + support: 'schema:Text', + CacheFile: 'pt:CacheFile' }, { likes: { diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts index 381a29e66..2562ead9b 100644 --- a/server/helpers/custom-validators/activitypub/activity.ts +++ b/server/helpers/custom-validators/activitypub/activity.ts @@ -1,7 +1,10 @@ import * as validator from 'validator' import { Activity, ActivityType } from '../../../../shared/models/activitypub' import { - isActorAcceptActivityValid, isActorDeleteActivityValid, isActorFollowActivityValid, isActorRejectActivityValid, + isActorAcceptActivityValid, + isActorDeleteActivityValid, + isActorFollowActivityValid, + isActorRejectActivityValid, isActorUpdateActivityValid } from './actor' import { isAnnounceActivityValid } from './announce' @@ -11,12 +14,13 @@ import { isUndoActivityValid } from './undo' import { isVideoCommentCreateActivityValid, isVideoCommentDeleteActivityValid } from './video-comments' import { isVideoFlagValid, - sanitizeAndCheckVideoTorrentCreateActivity, isVideoTorrentDeleteActivityValid, + sanitizeAndCheckVideoTorrentCreateActivity, sanitizeAndCheckVideoTorrentUpdateActivity } from './videos' import { isViewActivityValid } from './view' import { exists } from '../misc' +import { isCacheFileCreateActivityValid, isCacheFileUpdateActivityValid } from './cache-file' function isRootActivityValid (activity: any) { return Array.isArray(activity['@context']) && ( @@ -67,11 +71,13 @@ function checkCreateActivity (activity: any) { isDislikeActivityValid(activity) || sanitizeAndCheckVideoTorrentCreateActivity(activity) || isVideoFlagValid(activity) || - isVideoCommentCreateActivityValid(activity) + isVideoCommentCreateActivityValid(activity) || + isCacheFileCreateActivityValid(activity) } function checkUpdateActivity (activity: any) { - return sanitizeAndCheckVideoTorrentUpdateActivity(activity) || + return isCacheFileUpdateActivityValid(activity) || + sanitizeAndCheckVideoTorrentUpdateActivity(activity) || isActorUpdateActivityValid(activity) } diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts new file mode 100644 index 000000000..bd70934c8 --- /dev/null +++ b/server/helpers/custom-validators/activitypub/cache-file.ts @@ -0,0 +1,28 @@ +import { isActivityPubUrlValid, isBaseActivityValid } from './misc' +import { isRemoteVideoUrlValid } from './videos' +import { isDateValid, exists } from '../misc' +import { CacheFileObject } from '../../../../shared/models/activitypub/objects' + +function isCacheFileCreateActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Create') && + isCacheFileObjectValid(activity.object) +} + +function isCacheFileUpdateActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Update') && + isCacheFileObjectValid(activity.object) +} + +function isCacheFileObjectValid (object: CacheFileObject) { + return exists(object) && + object.type === 'CacheFile' && + isDateValid(object.expires) && + isActivityPubUrlValid(object.object) && + isRemoteVideoUrlValid(object.url) +} + +export { + isCacheFileUpdateActivityValid, + isCacheFileCreateActivityValid, + isCacheFileObjectValid +} diff --git a/server/helpers/custom-validators/activitypub/misc.ts b/server/helpers/custom-validators/activitypub/misc.ts index 6c5c7abca..4e2c57f04 100644 --- a/server/helpers/custom-validators/activitypub/misc.ts +++ b/server/helpers/custom-validators/activitypub/misc.ts @@ -3,7 +3,7 @@ import { CONSTRAINTS_FIELDS } from '../../../initializers' import { isTestInstance } from '../../core-utils' import { exists } from '../misc' -function isActivityPubUrlValid (url: string) { +function isUrlValid (url: string) { const isURLOptions = { require_host: true, require_tld: true, @@ -17,13 +17,18 @@ function isActivityPubUrlValid (url: string) { isURLOptions.require_tld = false } - return exists(url) && validator.isURL('' + url, isURLOptions) && validator.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL) + return exists(url) && validator.isURL('' + url, isURLOptions) +} + +function isActivityPubUrlValid (url: string) { + return isUrlValid(url) && validator.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL) } function isBaseActivityValid (activity: any, type: string) { return (activity['@context'] === undefined || Array.isArray(activity['@context'])) && activity.type === type && isActivityPubUrlValid(activity.id) && + exists(activity.actor) && (isActivityPubUrlValid(activity.actor) || isActivityPubUrlValid(activity.actor.id)) && ( activity.to === undefined || @@ -49,6 +54,7 @@ function setValidAttributedTo (obj: any) { } export { + isUrlValid, isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo diff --git a/server/helpers/custom-validators/activitypub/undo.ts b/server/helpers/custom-validators/activitypub/undo.ts index f50f224fa..578035893 100644 --- a/server/helpers/custom-validators/activitypub/undo.ts +++ b/server/helpers/custom-validators/activitypub/undo.ts @@ -2,6 +2,7 @@ import { isActorFollowActivityValid } from './actor' import { isBaseActivityValid } from './misc' import { isDislikeActivityValid, isLikeActivityValid } from './rate' import { isAnnounceActivityValid } from './announce' +import { isCacheFileCreateActivityValid } from './cache-file' function isUndoActivityValid (activity: any) { return isBaseActivityValid(activity, 'Undo') && @@ -9,7 +10,8 @@ function isUndoActivityValid (activity: any) { isActorFollowActivityValid(activity.object) || isLikeActivityValid(activity.object) || isDislikeActivityValid(activity.object) || - isAnnounceActivityValid(activity.object) + isAnnounceActivityValid(activity.object) || + isCacheFileCreateActivityValid(activity.object) ) } diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index 0362f43ab..f76eba474 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -75,6 +75,30 @@ function sanitizeAndCheckVideoTorrentObject (video: any) { video.attributedTo.length !== 0 } +function isRemoteVideoUrlValid (url: any) { + // FIXME: Old bug, we used the width to represent the resolution. Remove it in a few release (currently beta.11) + if (url.width && !url.height) url.height = url.width + + return url.type === 'Link' && + ( + ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mimeType) !== -1 && + isActivityPubUrlValid(url.href) && + validator.isInt(url.height + '', { min: 0 }) && + validator.isInt(url.size + '', { min: 0 }) && + (!url.fps || validator.isInt(url.fps + '', { min: 0 })) + ) || + ( + ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mimeType) !== -1 && + isActivityPubUrlValid(url.href) && + validator.isInt(url.height + '', { min: 0 }) + ) || + ( + ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mimeType) !== -1 && + validator.isLength(url.href, { min: 5 }) && + validator.isInt(url.height + '', { min: 0 }) + ) +} + // --------------------------------------------------------------------------- export { @@ -83,7 +107,8 @@ export { isVideoTorrentDeleteActivityValid, isRemoteStringIdentifierValid, isVideoFlagValid, - sanitizeAndCheckVideoTorrentObject + sanitizeAndCheckVideoTorrentObject, + isRemoteVideoUrlValid } // --------------------------------------------------------------------------- @@ -147,26 +172,4 @@ function setRemoteVideoTruncatedContent (video: any) { return true } -function isRemoteVideoUrlValid (url: any) { - // FIXME: Old bug, we used the width to represent the resolution. Remove it in a few realease (currently beta.11) - if (url.width && !url.height) url.height = url.width - return url.type === 'Link' && - ( - ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mimeType) !== -1 && - isActivityPubUrlValid(url.href) && - validator.isInt(url.height + '', { min: 0 }) && - validator.isInt(url.size + '', { min: 0 }) && - (!url.fps || validator.isInt(url.fps + '', { min: 0 })) - ) || - ( - ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mimeType) !== -1 && - isActivityPubUrlValid(url.href) && - validator.isInt(url.height + '', { min: 0 }) - ) || - ( - ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mimeType) !== -1 && - validator.isLength(url.href, { min: 5 }) && - validator.isInt(url.height + '', { min: 0 }) - ) -} diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index 1c0cc7058..2fdfd1876 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -5,44 +5,49 @@ import { createWriteStream, remove } from 'fs-extra' import { CONFIG } from '../initializers' import { join } from 'path' -function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: string }) { +function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: string }, timeout?: number) { const id = target.magnetUri || target.torrentName + let timer const path = generateVideoTmpPath(id) logger.info('Importing torrent video %s', id) return new Promise((res, rej) => { const webtorrent = new WebTorrent() + let file: WebTorrent.TorrentFile const torrentId = target.magnetUri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName) const options = { path: CONFIG.STORAGE.VIDEOS_DIR } const torrent = webtorrent.add(torrentId, options, torrent => { - if (torrent.files.length !== 1) return rej(new Error('The number of files is not equal to 1 for ' + torrentId)) + if (torrent.files.length !== 1) { + if (timer) clearTimeout(timer) - const file = torrent.files[ 0 ] + return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName) + .then(() => rej(new Error('The number of files is not equal to 1 for ' + torrentId))) + } + + file = torrent.files[ 0 ] const writeStream = createWriteStream(path) writeStream.on('finish', () => { - webtorrent.destroy(async err => { - if (err) return rej(err) - - if (target.torrentName) { - remove(torrentId) - .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err })) - } + if (timer) clearTimeout(timer) - remove(join(CONFIG.STORAGE.VIDEOS_DIR, file.name)) - .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', file.name, { err })) - - res(path) - }) + return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName) + .then(() => res(path)) }) file.createReadStream().pipe(writeStream) }) torrent.on('error', err => rej(err)) + + if (timeout) { + timer = setTimeout(async () => { + return safeWebtorrentDestroy(webtorrent, torrentId, file ? file.name : undefined, target.torrentName) + .then(() => rej(new Error('Webtorrent download timeout.'))) + }, timeout) + } }) } @@ -51,3 +56,29 @@ function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: stri export { downloadWebTorrentVideo } + +// --------------------------------------------------------------------------- + +function safeWebtorrentDestroy (webtorrent: WebTorrent.Instance, torrentId: string, filename?: string, torrentName?: string) { + return new Promise(res => { + webtorrent.destroy(err => { + // Delete torrent file + if (torrentName) { + remove(torrentId) + .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err })) + } + + // Delete downloaded file + if (filename) { + remove(join(CONFIG.STORAGE.VIDEOS_DIR, filename)) + .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', filename, { err })) + } + + if (err) { + logger.warn('Cannot destroy webtorrent in timeout.', { err }) + } + + return res() + }) + }) +} diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts index 9dd104035..6a2badd35 100644 --- a/server/initializers/checker.ts +++ b/server/initializers/checker.ts @@ -7,6 +7,9 @@ import { parse } from 'url' import { CONFIG } from './constants' import { logger } from '../helpers/logger' import { getServerActor } from '../helpers/utils' +import { VideosRedundancy } from '../../shared/models/redundancy' +import { isArray } from '../helpers/custom-validators/misc' +import { uniq } from 'lodash' async function checkActivityPubUrls () { const actor = await getServerActor() @@ -35,6 +38,20 @@ function checkConfig () { return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy } + const redundancyVideos = config.get('redundancy.videos') + if (isArray(redundancyVideos)) { + for (const r of redundancyVideos) { + if ([ 'most-views' ].indexOf(r.strategy) === -1) { + return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy + } + } + + const filtered = uniq(redundancyVideos.map(r => r.strategy)) + if (filtered.length !== redundancyVideos.length) { + return 'Redundancy video entries should have uniq strategies' + } + } + return null } diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 5b7ea5d6c..6b4afbfd8 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -1,6 +1,6 @@ import { IConfig } from 'config' import { dirname, join } from 'path' -import { JobType, VideoRateType, VideoState } from '../../shared/models' +import { JobType, VideoRateType, VideoRedundancyStrategy, VideoState, VideosRedundancy } from '../../shared/models' import { ActivityPubActorType } from '../../shared/models/activitypub' import { FollowState } from '../../shared/models/actors' import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos' @@ -9,13 +9,14 @@ import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../h import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' import { invert } from 'lodash' import { CronRepeatOptions, EveryRepeatOptions } from 'bull' +import * as bytes from 'bytes' // Use a variable to reload the configuration if we need let config: IConfig = require('config') // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 265 +const LAST_MIGRATION_VERSION = 270 // --------------------------------------------------------------------------- @@ -137,7 +138,8 @@ let SCHEDULER_INTERVALS_MS = { badActorFollow: 60000 * 60, // 1 hour removeOldJobs: 60000 * 60, // 1 hour updateVideos: 60000, // 1 minute - youtubeDLUpdate: 60000 * 60 * 24 // 1 day + youtubeDLUpdate: 60000 * 60 * 24, // 1 day + videosRedundancy: 60000 * 2 // 2 hours } // --------------------------------------------------------------------------- @@ -208,6 +210,9 @@ const CONFIG = { INTERVAL_DAYS: config.get('trending.videos.interval_days') } }, + REDUNDANCY: { + VIDEOS: buildVideosRedundancy(config.get('redundancy.videos')) + }, ADMIN: { get EMAIL () { return config.get('admin.email') } }, @@ -321,6 +326,9 @@ const CONSTRAINTS_FIELDS = { } } }, + VIDEOS_REDUNDANCY: { + URL: { min: 3, max: 2000 } // Length + }, VIDEOS: { NAME: { min: 3, max: 120 }, // Length LANGUAGE: { min: 1, max: 10 }, // Length @@ -584,6 +592,13 @@ const CACHE = { } } +const REDUNDANCY = { + VIDEOS: { + EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days + RANDOMIZED_FACTOR: 5 + } +} + const ACCEPT_HEADERS = [ 'html', 'application/json' ].concat(ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS) // --------------------------------------------------------------------------- @@ -629,8 +644,11 @@ if (isTestInstance() === true) { SCHEDULER_INTERVALS_MS.badActorFollow = 10000 SCHEDULER_INTERVALS_MS.removeOldJobs = 10000 SCHEDULER_INTERVALS_MS.updateVideos = 5000 + SCHEDULER_INTERVALS_MS.videosRedundancy = 5000 REPEAT_JOBS['videos-views'] = { every: 5000 } + REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1 + VIDEO_VIEW_LIFETIME = 1000 // 1 second JOB_ATTEMPTS['email'] = 1 @@ -653,6 +671,7 @@ export { CONFIG, CONSTRAINTS_FIELDS, EMBED_SIZE, + REDUNDANCY, JOB_CONCURRENCY, JOB_ATTEMPTS, LAST_MIGRATION_VERSION, @@ -722,6 +741,17 @@ function updateWebserverConfig () { CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP) } +function buildVideosRedundancy (objs: { strategy: VideoRedundancyStrategy, size: string }[]): VideosRedundancy[] { + if (!objs) return [] + + return objs.map(obj => { + return { + strategy: obj.strategy, + size: bytes.parse(obj.size) + } + }) +} + function buildLanguages () { const iso639 = require('iso-639-3') diff --git a/server/initializers/database.ts b/server/initializers/database.ts index b68e1a882..4d57bf8aa 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts @@ -27,6 +27,7 @@ import { VideoCaptionModel } from '../models/video/video-caption' import { VideoImportModel } from '../models/video/video-import' import { VideoViewModel } from '../models/video/video-views' import { VideoChangeOwnershipModel } from '../models/video/video-change-ownership' +import { VideoRedundancyModel } from '../models/redundancy/video-redundancy' require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string @@ -87,7 +88,8 @@ async function initDatabaseModels (silent: boolean) { VideoCommentModel, ScheduleVideoUpdateModel, VideoImportModel, - VideoViewModel + VideoViewModel, + VideoRedundancyModel ]) // Check extensions exist in the database diff --git a/server/initializers/migrations/0270-server-redundancy.ts b/server/initializers/migrations/0270-server-redundancy.ts new file mode 100644 index 000000000..903ba8a85 --- /dev/null +++ b/server/initializers/migrations/0270-server-redundancy.ts @@ -0,0 +1,24 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize +}): Promise { + { + const data = { + type: Sequelize.BOOLEAN, + allowNull: false, + defaultValue: false + } + + await utils.queryInterface.addColumn('server', 'redundancyAllowed', data) + } + +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { up, down } diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 1657262d7..3464add03 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -400,17 +400,15 @@ async function refreshActorIfNeeded (actor: ActorModel): Promise<{ actor: ActorM await actor.save({ transaction: t }) if (actor.Account) { - await actor.save({ transaction: t }) - actor.Account.set('name', result.name) actor.Account.set('description', result.summary) + await actor.Account.save({ transaction: t }) } else if (actor.VideoChannel) { - await actor.save({ transaction: t }) - actor.VideoChannel.set('name', result.name) actor.VideoChannel.set('description', result.summary) actor.VideoChannel.set('support', result.support) + await actor.VideoChannel.save({ transaction: t }) } diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts new file mode 100644 index 000000000..7325ddcb6 --- /dev/null +++ b/server/lib/activitypub/cache-file.ts @@ -0,0 +1,47 @@ +import { CacheFileObject } from '../../../shared/index' +import { VideoModel } from '../../models/video/video' +import { ActorModel } from '../../models/activitypub/actor' +import { sequelizeTypescript } from '../../initializers' +import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' + +function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) { + const url = cacheFileObject.url + + const videoFile = video.VideoFiles.find(f => { + return f.resolution === url.height && f.fps === url.fps + }) + + if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`) + + return { + expiresOn: new Date(cacheFileObject.expires), + url: cacheFileObject.id, + fileUrl: cacheFileObject.url.href, + strategy: null, + videoFileId: videoFile.id, + actorId: byActor.id + } +} + +function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) { + return sequelizeTypescript.transaction(async t => { + const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) + + return VideoRedundancyModel.create(attributes, { transaction: t }) + }) +} + +function updateCacheFile (cacheFileObject: CacheFileObject, redundancyModel: VideoRedundancyModel, byActor: ActorModel) { + const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, redundancyModel.VideoFile.Video, byActor) + + redundancyModel.set('expires', attributes.expiresOn) + redundancyModel.set('fileUrl', attributes.fileUrl) + + return redundancyModel.save() +} + +export { + createCacheFile, + updateCacheFile, + cacheFileActivityObjectToDBAttributes +} diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 16f426e23..32e555acf 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -1,4 +1,4 @@ -import { ActivityCreate, VideoAbuseState, VideoTorrentObject } from '../../../../shared' +import { ActivityCreate, CacheFileObject, VideoAbuseState, VideoTorrentObject } from '../../../../shared' import { DislikeObject, VideoAbuseObject, ViewObject } from '../../../../shared/models/activitypub/objects' import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object' import { retryTransactionWrapper } from '../../../helpers/database-utils' @@ -12,6 +12,7 @@ import { addVideoComment, resolveThread } from '../video-comments' import { getOrCreateVideoAndAccountAndChannel } from '../videos' import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils' import { Redis } from '../../redis' +import { createCacheFile } from '../cache-file' async function processCreateActivity (activity: ActivityCreate) { const activityObject = activity.object @@ -28,6 +29,8 @@ async function processCreateActivity (activity: ActivityCreate) { return retryTransactionWrapper(processCreateVideoAbuse, actor, activityObject as VideoAbuseObject) } else if (activityType === 'Note') { return retryTransactionWrapper(processCreateVideoComment, actor, activity) + } else if (activityType === 'CacheFile') { + return retryTransactionWrapper(processCacheFile, actor, activity) } logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) @@ -97,6 +100,20 @@ async function processCreateView (byActor: ActorModel, activity: ActivityCreate) } } +async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) { + const cacheFile = activity.object as CacheFileObject + + const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFile.object) + + await createCacheFile(cacheFile, video, byActor) + + if (video.isOwned()) { + // Don't resend the activity to the sender + const exceptions = [ byActor ] + await forwardActivity(activity, undefined, exceptions) + } +} + async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) { logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object) @@ -113,7 +130,7 @@ async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateDat state: VideoAbuseState.PENDING } - await VideoAbuseModel.create(videoAbuseData) + await VideoAbuseModel.create(videoAbuseData, { transaction: t }) logger.info('Remote abuse for video uuid %s created', videoAbuseToCreateData.object) }) diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index 1c1de8827..0eb5fa392 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts @@ -1,4 +1,4 @@ -import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo } from '../../../../shared/models/activitypub' +import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo, CacheFileObject } from '../../../../shared/models/activitypub' import { DislikeObject } from '../../../../shared/models/activitypub/objects' import { getActorUrl } from '../../../helpers/activitypub' import { retryTransactionWrapper } from '../../../helpers/database-utils' @@ -11,6 +11,7 @@ import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { forwardVideoRelatedActivity } from '../send/utils' import { getOrCreateVideoAndAccountAndChannel } from '../videos' import { VideoShareModel } from '../../../models/video/video-share' +import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' async function processUndoActivity (activity: ActivityUndo) { const activityToUndo = activity.object @@ -19,11 +20,21 @@ async function processUndoActivity (activity: ActivityUndo) { if (activityToUndo.type === 'Like') { return retryTransactionWrapper(processUndoLike, actorUrl, activity) - } else if (activityToUndo.type === 'Create' && activityToUndo.object.type === 'Dislike') { - return retryTransactionWrapper(processUndoDislike, actorUrl, activity) - } else if (activityToUndo.type === 'Follow') { + } + + if (activityToUndo.type === 'Create') { + if (activityToUndo.object.type === 'Dislike') { + return retryTransactionWrapper(processUndoDislike, actorUrl, activity) + } else if (activityToUndo.object.type === 'CacheFile') { + return retryTransactionWrapper(processUndoCacheFile, actorUrl, activity) + } + } + + if (activityToUndo.type === 'Follow') { return retryTransactionWrapper(processUndoFollow, actorUrl, activityToUndo) - } else if (activityToUndo.type === 'Announce') { + } + + if (activityToUndo.type === 'Announce') { return retryTransactionWrapper(processUndoAnnounce, actorUrl, activityToUndo) } @@ -88,6 +99,29 @@ async function processUndoDislike (actorUrl: string, activity: ActivityUndo) { }) } +async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) { + const cacheFileObject = activity.object.object as CacheFileObject + + const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.object) + + return sequelizeTypescript.transaction(async t => { + const byActor = await ActorModel.loadByUrl(actorUrl) + if (!byActor) throw new Error('Unknown actor ' + actorUrl) + + const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) + if (!cacheFile) throw new Error('Unknown video cache ' + cacheFile.url) + + await cacheFile.destroy() + + if (video.isOwned()) { + // Don't resend the activity to the sender + const exceptions = [ byActor ] + + await forwardVideoRelatedActivity(activity, t, exceptions, video) + } + }) +} + function processUndoFollow (actorUrl: string, followActivity: ActivityFollow) { return sequelizeTypescript.transaction(async t => { const follower = await ActorModel.loadByUrl(actorUrl, t) diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index d2ad738a2..d3af1a181 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -1,4 +1,4 @@ -import { ActivityUpdate, VideoTorrentObject } from '../../../../shared/models/activitypub' +import { ActivityUpdate, CacheFileObject, VideoTorrentObject } from '../../../../shared/models/activitypub' import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor' import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils' import { logger } from '../../../helpers/logger' @@ -7,8 +7,11 @@ import { AccountModel } from '../../../models/account/account' import { ActorModel } from '../../../models/activitypub/actor' import { VideoChannelModel } from '../../../models/video/video-channel' import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor' -import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos' +import { getOrCreateVideoAndAccountAndChannel, updateVideoFromAP, getOrCreateVideoChannelFromVideoObject } from '../videos' import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' +import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' +import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' +import { createCacheFile, updateCacheFile } from '../cache-file' async function processUpdateActivity (activity: ActivityUpdate) { const actor = await getOrCreateActorAndServerAndModel(activity.actor) @@ -16,10 +19,16 @@ async function processUpdateActivity (activity: ActivityUpdate) { if (objectType === 'Video') { return retryTransactionWrapper(processUpdateVideo, actor, activity) - } else if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') { + } + + if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') { return retryTransactionWrapper(processUpdateActor, actor, activity) } + if (objectType === 'CacheFile') { + return retryTransactionWrapper(processUpdateCacheFile, actor, activity) + } + return undefined } @@ -42,7 +51,24 @@ async function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id) const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) - return updateVideoFromAP(video, videoObject, actor, channelActor, activity.to) + return updateVideoFromAP(video, videoObject, actor.Account, channelActor.VideoChannel, activity.to) +} + +async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) { + const cacheFileObject = activity.object as CacheFileObject + + if (!isCacheFileObjectValid(cacheFileObject) === false) { + logger.debug('Cahe file object sent by update is not valid.', { cacheFileObject }) + return undefined + } + + const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id) + if (!redundancyModel) { + const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.id) + return createCacheFile(cacheFileObject, video, byActor) + } + + return updateCacheFile(cacheFileObject, redundancyModel, byActor) } async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) { diff --git a/server/lib/activitypub/send/send-accept.ts b/server/lib/activitypub/send/send-accept.ts index ef679707b..b6abde13d 100644 --- a/server/lib/activitypub/send/send-accept.ts +++ b/server/lib/activitypub/send/send-accept.ts @@ -3,7 +3,7 @@ import { ActorModel } from '../../../models/activitypub/actor' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { getActorFollowAcceptActivityPubUrl, getActorFollowActivityPubUrl } from '../url' import { unicastTo } from './utils' -import { followActivityData } from './send-follow' +import { buildFollowActivity } from './send-follow' import { logger } from '../../../helpers/logger' async function sendAccept (actorFollow: ActorFollowModel) { @@ -18,10 +18,10 @@ async function sendAccept (actorFollow: ActorFollowModel) { logger.info('Creating job to accept follower %s.', follower.url) const followUrl = getActorFollowActivityPubUrl(actorFollow) - const followData = followActivityData(followUrl, follower, me) + const followData = buildFollowActivity(followUrl, follower, me) const url = getActorFollowAcceptActivityPubUrl(actorFollow) - const data = acceptActivityData(url, me, followData) + const data = buildAcceptActivity(url, me, followData) return unicastTo(data, me, follower.inboxUrl) } @@ -34,7 +34,7 @@ export { // --------------------------------------------------------------------------- -function acceptActivityData (url: string, byActor: ActorModel, followActivityData: ActivityFollow): ActivityAccept { +function buildAcceptActivity (url: string, byActor: ActorModel, followActivityData: ActivityFollow): ActivityAccept { return { type: 'Accept', id: url, diff --git a/server/lib/activitypub/send/send-announce.ts b/server/lib/activitypub/send/send-announce.ts index 352813d73..f137217f8 100644 --- a/server/lib/activitypub/send/send-announce.ts +++ b/server/lib/activitypub/send/send-announce.ts @@ -4,45 +4,44 @@ import { ActorModel } from '../../../models/activitypub/actor' import { VideoModel } from '../../../models/video/video' import { VideoShareModel } from '../../../models/video/video-share' import { broadcastToFollowers } from './utils' -import { getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience' +import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience' import { logger } from '../../../helpers/logger' -async function buildVideoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { +async function buildAnnounceWithVideoAudience (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { const announcedObject = video.url - const accountsToForwardView = await getActorsInvolvedInVideo(video, t) - const audience = getObjectFollowersAudience(accountsToForwardView) - return announceActivityData(videoShare.url, byActor, announcedObject, audience) + const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) + const audience = getObjectFollowersAudience(actorsInvolvedInVideo) + + const activity = buildAnnounceActivity(videoShare.url, byActor, announcedObject, audience) + + return { activity, actorsInvolvedInVideo } } async function sendVideoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { - const data = await buildVideoAnnounce(byActor, videoShare, video, t) + const { activity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, t) logger.info('Creating job to send announce %s.', videoShare.url) - const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) const followersException = [ byActor ] - - return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException) + return broadcastToFollowers(activity, byActor, actorsInvolvedInVideo, t, followersException) } -function announceActivityData (url: string, byActor: ActorModel, object: string, audience?: ActivityAudience): ActivityAnnounce { +function buildAnnounceActivity (url: string, byActor: ActorModel, object: string, audience?: ActivityAudience): ActivityAnnounce { if (!audience) audience = getAudience(byActor) - return { - type: 'Announce', - to: audience.to, - cc: audience.cc, + return audiencify({ + type: 'Announce' as 'Announce', id: url, actor: byActor.url, object - } + }, audience) } // --------------------------------------------------------------------------- export { sendVideoAnnounce, - announceActivityData, - buildVideoAnnounce + buildAnnounceActivity, + buildAnnounceWithVideoAudience } diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts index fc76cdd8a..6f89b1a22 100644 --- a/server/lib/activitypub/send/send-create.ts +++ b/server/lib/activitypub/send/send-create.ts @@ -17,6 +17,7 @@ import { getVideoCommentAudience } from '../audience' import { logger } from '../../../helpers/logger' +import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' async function sendCreateVideo (video: VideoModel, t: Transaction) { if (video.privacy === VideoPrivacy.PRIVATE) return undefined @@ -27,12 +28,12 @@ async function sendCreateVideo (video: VideoModel, t: Transaction) { const videoObject = video.toActivityPubObject() const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) - const data = createActivityData(video.url, byActor, videoObject, audience) + const createActivity = buildCreateActivity(video.url, byActor, videoObject, audience) - return broadcastToFollowers(data, byActor, [ byActor ], t) + return broadcastToFollowers(createActivity, byActor, [ byActor ], t) } -async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel, t: Transaction) { +async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) { if (!video.VideoChannel.Account.Actor.serverId) return // Local const url = getVideoAbuseActivityPubUrl(videoAbuse) @@ -40,9 +41,23 @@ async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, logger.info('Creating job to send video abuse %s.', url) const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] } - const data = createActivityData(url, byActor, videoAbuse.toActivityPubObject(), audience) + const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience) - return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) + return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) +} + +async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) { + logger.info('Creating job to send file cache of %s.', fileRedundancy.url) + + const redundancyObject = fileRedundancy.toActivityPubObject() + + const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id) + const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined) + + const audience = getVideoAudience(video, actorsInvolvedInVideo) + const createActivity = buildCreateActivity(fileRedundancy.url, byActor, redundancyObject, audience) + + return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) } async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) { @@ -66,73 +81,73 @@ async function sendCreateVideoComment (comment: VideoCommentModel, t: Transactio audience = getObjectFollowersAudience(actorsInvolvedInComment.concat(parentsCommentActors)) } - const data = createActivityData(comment.url, byActor, commentObject, audience) + const createActivity = buildCreateActivity(comment.url, byActor, commentObject, audience) // This was a reply, send it to the parent actors const actorsException = [ byActor ] - await broadcastToActors(data, byActor, parentsCommentActors, actorsException) + await broadcastToActors(createActivity, byActor, parentsCommentActors, actorsException) // Broadcast to our followers - await broadcastToFollowers(data, byActor, [ byActor ], t) + await broadcastToFollowers(createActivity, byActor, [ byActor ], t) // Send to actors involved in the comment - if (isOrigin) return broadcastToFollowers(data, byActor, actorsInvolvedInComment, t, actorsException) + if (isOrigin) return broadcastToFollowers(createActivity, byActor, actorsInvolvedInComment, t, actorsException) // Send to origin - return unicastTo(data, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl) + return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl) } async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transaction) { logger.info('Creating job to send view of %s.', video.url) const url = getVideoViewActivityPubUrl(byActor, video) - const viewActivityData = createViewActivityData(byActor, video) + const viewActivity = buildViewActivity(byActor, video) const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) // Send to origin if (video.isOwned() === false) { const audience = getVideoAudience(video, actorsInvolvedInVideo) - const data = createActivityData(url, byActor, viewActivityData, audience) + const createActivity = buildCreateActivity(url, byActor, viewActivity, audience) - return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) + return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) } // Send to followers const audience = getObjectFollowersAudience(actorsInvolvedInVideo) - const data = createActivityData(url, byActor, viewActivityData, audience) + const createActivity = buildCreateActivity(url, byActor, viewActivity, audience) // Use the server actor to send the view const serverActor = await getServerActor() const actorsException = [ byActor ] - return broadcastToFollowers(data, serverActor, actorsInvolvedInVideo, t, actorsException) + return broadcastToFollowers(createActivity, serverActor, actorsInvolvedInVideo, t, actorsException) } async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { logger.info('Creating job to dislike %s.', video.url) const url = getVideoDislikeActivityPubUrl(byActor, video) - const dislikeActivityData = createDislikeActivityData(byActor, video) + const dislikeActivity = buildDislikeActivity(byActor, video) const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) // Send to origin if (video.isOwned() === false) { const audience = getVideoAudience(video, actorsInvolvedInVideo) - const data = createActivityData(url, byActor, dislikeActivityData, audience) + const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience) - return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) + return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) } // Send to followers const audience = getObjectFollowersAudience(actorsInvolvedInVideo) - const data = createActivityData(url, byActor, dislikeActivityData, audience) + const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience) const actorsException = [ byActor ] - return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, actorsException) + return broadcastToFollowers(createActivity, byActor, actorsInvolvedInVideo, t, actorsException) } -function createActivityData (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { +function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate { if (!audience) audience = getAudience(byActor) return audiencify( @@ -146,7 +161,7 @@ function createActivityData (url: string, byActor: ActorModel, object: any, audi ) } -function createDislikeActivityData (byActor: ActorModel, video: VideoModel) { +function buildDislikeActivity (byActor: ActorModel, video: VideoModel) { return { type: 'Dislike', actor: byActor.url, @@ -154,7 +169,7 @@ function createDislikeActivityData (byActor: ActorModel, video: VideoModel) { } } -function createViewActivityData (byActor: ActorModel, video: VideoModel) { +function buildViewActivity (byActor: ActorModel, video: VideoModel) { return { type: 'View', actor: byActor.url, @@ -167,9 +182,10 @@ function createViewActivityData (byActor: ActorModel, video: VideoModel) { export { sendCreateVideo, sendVideoAbuse, - createActivityData, + buildCreateActivity, sendCreateView, sendCreateDislike, - createDislikeActivityData, - sendCreateVideoComment + buildDislikeActivity, + sendCreateVideoComment, + sendCreateCacheFile } diff --git a/server/lib/activitypub/send/send-delete.ts b/server/lib/activitypub/send/send-delete.ts index 3d1dfb699..479182543 100644 --- a/server/lib/activitypub/send/send-delete.ts +++ b/server/lib/activitypub/send/send-delete.ts @@ -15,24 +15,23 @@ async function sendDeleteVideo (video: VideoModel, t: Transaction) { const url = getDeleteActivityPubUrl(video.url) const byActor = video.VideoChannel.Account.Actor - const data = deleteActivityData(url, video.url, byActor) + const activity = buildDeleteActivity(url, video.url, byActor) - const actorsInvolved = await VideoShareModel.loadActorsByShare(video.id, t) - actorsInvolved.push(byActor) + const actorsInvolved = await getActorsInvolvedInVideo(video, t) - return broadcastToFollowers(data, byActor, actorsInvolved, t) + return broadcastToFollowers(activity, byActor, actorsInvolved, t) } async function sendDeleteActor (byActor: ActorModel, t: Transaction) { logger.info('Creating job to broadcast delete of actor %s.', byActor.url) const url = getDeleteActivityPubUrl(byActor.url) - const data = deleteActivityData(url, byActor.url, byActor) + const activity = buildDeleteActivity(url, byActor.url, byActor) const actorsInvolved = await VideoShareModel.loadActorsByVideoOwner(byActor.id, t) actorsInvolved.push(byActor) - return broadcastToFollowers(data, byActor, actorsInvolved, t) + return broadcastToFollowers(activity, byActor, actorsInvolved, t) } async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Transaction) { @@ -45,23 +44,23 @@ async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Trans const threadParentComments = await VideoCommentModel.listThreadParentComments(videoComment, t) const actorsInvolvedInComment = await getActorsInvolvedInVideo(videoComment.Video, t) - actorsInvolvedInComment.push(byActor) + actorsInvolvedInComment.push(byActor) // Add the actor that commented the video const audience = getVideoCommentAudience(videoComment, threadParentComments, actorsInvolvedInComment, isVideoOrigin) - const data = deleteActivityData(url, videoComment.url, byActor, audience) + const activity = buildDeleteActivity(url, videoComment.url, byActor, audience) // This was a reply, send it to the parent actors const actorsException = [ byActor ] - await broadcastToActors(data, byActor, threadParentComments.map(c => c.Account.Actor), actorsException) + await broadcastToActors(activity, byActor, threadParentComments.map(c => c.Account.Actor), actorsException) // Broadcast to our followers - await broadcastToFollowers(data, byActor, [ byActor ], t) + await broadcastToFollowers(activity, byActor, [ byActor ], t) // Send to actors involved in the comment - if (isVideoOrigin) return broadcastToFollowers(data, byActor, actorsInvolvedInComment, t, actorsException) + if (isVideoOrigin) return broadcastToFollowers(activity, byActor, actorsInvolvedInComment, t, actorsException) // Send to origin - return unicastTo(data, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl) + return unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl) } // --------------------------------------------------------------------------- @@ -74,7 +73,7 @@ export { // --------------------------------------------------------------------------- -function deleteActivityData (url: string, object: string, byActor: ActorModel, audience?: ActivityAudience): ActivityDelete { +function buildDeleteActivity (url: string, object: string, byActor: ActorModel, audience?: ActivityAudience): ActivityDelete { const activity = { type: 'Delete' as 'Delete', id: url, diff --git a/server/lib/activitypub/send/send-follow.ts b/server/lib/activitypub/send/send-follow.ts index 46d08c17b..170b46b48 100644 --- a/server/lib/activitypub/send/send-follow.ts +++ b/server/lib/activitypub/send/send-follow.ts @@ -15,12 +15,12 @@ function sendFollow (actorFollow: ActorFollowModel) { logger.info('Creating job to send follow request to %s.', following.url) const url = getActorFollowActivityPubUrl(actorFollow) - const data = followActivityData(url, me, following) + const data = buildFollowActivity(url, me, following) return unicastTo(data, me, following.inboxUrl) } -function followActivityData (url: string, byActor: ActorModel, targetActor: ActorModel): ActivityFollow { +function buildFollowActivity (url: string, byActor: ActorModel, targetActor: ActorModel): ActivityFollow { return { type: 'Follow', id: url, @@ -33,5 +33,5 @@ function followActivityData (url: string, byActor: ActorModel, targetActor: Acto export { sendFollow, - followActivityData + buildFollowActivity } diff --git a/server/lib/activitypub/send/send-like.ts b/server/lib/activitypub/send/send-like.ts index 83225f5df..a5408ac6a 100644 --- a/server/lib/activitypub/send/send-like.ts +++ b/server/lib/activitypub/send/send-like.ts @@ -17,20 +17,20 @@ async function sendLike (byActor: ActorModel, video: VideoModel, t: Transaction) // Send to origin if (video.isOwned() === false) { const audience = getVideoAudience(video, accountsInvolvedInVideo) - const data = likeActivityData(url, byActor, video, audience) + const data = buildLikeActivity(url, byActor, video, audience) return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) } // Send to followers const audience = getObjectFollowersAudience(accountsInvolvedInVideo) - const data = likeActivityData(url, byActor, video, audience) + const activity = buildLikeActivity(url, byActor, video, audience) const followersException = [ byActor ] - return broadcastToFollowers(data, byActor, accountsInvolvedInVideo, t, followersException) + return broadcastToFollowers(activity, byActor, accountsInvolvedInVideo, t, followersException) } -function likeActivityData (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityLike { +function buildLikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityLike { if (!audience) audience = getAudience(byActor) return audiencify( @@ -48,5 +48,5 @@ function likeActivityData (url: string, byActor: ActorModel, video: VideoModel, export { sendLike, - likeActivityData + buildLikeActivity } diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts index 30d0fd98b..a50673c79 100644 --- a/server/lib/activitypub/send/send-undo.ts +++ b/server/lib/activitypub/send/send-undo.ts @@ -13,12 +13,13 @@ import { VideoModel } from '../../../models/video/video' import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url' import { broadcastToFollowers, unicastTo } from './utils' import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience, getVideoAudience } from '../audience' -import { createActivityData, createDislikeActivityData } from './send-create' -import { followActivityData } from './send-follow' -import { likeActivityData } from './send-like' +import { buildCreateActivity, buildDislikeActivity } from './send-create' +import { buildFollowActivity } from './send-follow' +import { buildLikeActivity } from './send-like' import { VideoShareModel } from '../../../models/video/video-share' -import { buildVideoAnnounce } from './send-announce' +import { buildAnnounceWithVideoAudience } from './send-announce' import { logger } from '../../../helpers/logger' +import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { const me = actorFollow.ActorFollower @@ -32,10 +33,10 @@ async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) { const followUrl = getActorFollowActivityPubUrl(actorFollow) const undoUrl = getUndoActivityPubUrl(followUrl) - const object = followActivityData(followUrl, me, following) - const data = undoActivityData(undoUrl, me, object) + const followActivity = buildFollowActivity(followUrl, me, following) + const undoActivity = undoActivityData(undoUrl, me, followActivity) - return unicastTo(data, me, following.inboxUrl) + return unicastTo(undoActivity, me, following.inboxUrl) } async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transaction) { @@ -45,21 +46,21 @@ async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transact const undoUrl = getUndoActivityPubUrl(likeUrl) const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) - const object = likeActivityData(likeUrl, byActor, video) + const likeActivity = buildLikeActivity(likeUrl, byActor, video) // Send to origin if (video.isOwned() === false) { const audience = getVideoAudience(video, actorsInvolvedInVideo) - const data = undoActivityData(undoUrl, byActor, object, audience) + const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience) - return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) + return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) } const audience = getObjectFollowersAudience(actorsInvolvedInVideo) - const data = undoActivityData(undoUrl, byActor, object, audience) + const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience) const followersException = [ byActor ] - return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException) + return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException) } async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Transaction) { @@ -69,20 +70,20 @@ async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Trans const undoUrl = getUndoActivityPubUrl(dislikeUrl) const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) - const dislikeActivity = createDislikeActivityData(byActor, video) - const object = createActivityData(dislikeUrl, byActor, dislikeActivity) + const dislikeActivity = buildDislikeActivity(byActor, video) + const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity) if (video.isOwned() === false) { const audience = getVideoAudience(video, actorsInvolvedInVideo) - const data = undoActivityData(undoUrl, byActor, object, audience) + const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity, audience) - return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) + return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) } - const data = undoActivityData(undoUrl, byActor, object) + const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity) const followersException = [ byActor ] - return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException) + return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException) } async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) { @@ -90,12 +91,27 @@ async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareMode const undoUrl = getUndoActivityPubUrl(videoShare.url) - const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) - const object = await buildVideoAnnounce(byActor, videoShare, video, t) - const data = undoActivityData(undoUrl, byActor, object) + const { activity: announceActivity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, t) + const undoActivity = undoActivityData(undoUrl, byActor, announceActivity) const followersException = [ byActor ] - return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException) + return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException) +} + +async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) { + logger.info('Creating job to undo cache file %s.', redundancyModel.url) + + const undoUrl = getUndoActivityPubUrl(redundancyModel.url) + + const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) + const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) + + const audience = getVideoAudience(video, actorsInvolvedInVideo) + const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) + + const undoActivity = undoActivityData(undoUrl, byActor, createActivity, audience) + + return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) } // --------------------------------------------------------------------------- @@ -104,7 +120,8 @@ export { sendUndoFollow, sendUndoLike, sendUndoDislike, - sendUndoAnnounce + sendUndoAnnounce, + sendUndoCacheFile } // --------------------------------------------------------------------------- diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts index 6f1d80898..605473338 100644 --- a/server/lib/activitypub/send/send-update.ts +++ b/server/lib/activitypub/send/send-update.ts @@ -7,11 +7,11 @@ import { VideoModel } from '../../../models/video/video' import { VideoChannelModel } from '../../../models/video/video-channel' import { VideoShareModel } from '../../../models/video/video-share' import { getUpdateActivityPubUrl } from '../url' -import { broadcastToFollowers } from './utils' -import { audiencify, getAudience } from '../audience' +import { broadcastToFollowers, unicastTo } from './utils' +import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience' import { logger } from '../../../helpers/logger' -import { videoFeedsValidator } from '../../../middlewares/validators' import { VideoCaptionModel } from '../../../models/video/video-caption' +import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByActor?: ActorModel) { logger.info('Creating job to update video %s.', video.url) @@ -26,12 +26,12 @@ async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByAct const videoObject = video.toActivityPubObject() const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) - const data = updateActivityData(url, byActor, videoObject, audience) + const updateActivity = buildUpdateActivity(url, byActor, videoObject, audience) - const actorsInvolved = await VideoShareModel.loadActorsByShare(video.id, t) - actorsInvolved.push(byActor) + const actorsInvolved = await getActorsInvolvedInVideo(video, t) + if (overrodeByActor) actorsInvolved.push(overrodeByActor) - return broadcastToFollowers(data, byActor, actorsInvolved, t) + return broadcastToFollowers(updateActivity, byActor, actorsInvolved, t) } async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelModel, t: Transaction) { @@ -42,7 +42,7 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod const url = getUpdateActivityPubUrl(byActor.url, byActor.updatedAt.toISOString()) const accountOrChannelObject = accountOrChannel.toActivityPubObject() const audience = getAudience(byActor) - const data = updateActivityData(url, byActor, accountOrChannelObject, audience) + const updateActivity = buildUpdateActivity(url, byActor, accountOrChannelObject, audience) let actorsInvolved: ActorModel[] if (accountOrChannel instanceof AccountModel) { @@ -55,19 +55,35 @@ async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelMod actorsInvolved.push(byActor) - return broadcastToFollowers(data, byActor, actorsInvolved, t) + return broadcastToFollowers(updateActivity, byActor, actorsInvolved, t) +} + +async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) { + logger.info('Creating job to update cache file %s.', redundancyModel.url) + + const url = getUpdateActivityPubUrl(redundancyModel.url, redundancyModel.updatedAt.toISOString()) + const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id) + + const redundancyObject = redundancyModel.toActivityPubObject() + + const accountsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined) + const audience = getObjectFollowersAudience(accountsInvolvedInVideo) + + const updateActivity = buildUpdateActivity(url, byActor, redundancyObject, audience) + return unicastTo(updateActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl) } // --------------------------------------------------------------------------- export { sendUpdateActor, - sendUpdateVideo + sendUpdateVideo, + sendUpdateCacheFile } // --------------------------------------------------------------------------- -function updateActivityData (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityUpdate { +function buildUpdateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityUpdate { if (!audience) audience = getAudience(byActor) return audiencify( diff --git a/server/lib/activitypub/send/utils.ts b/server/lib/activitypub/send/utils.ts index da437292e..c20c15633 100644 --- a/server/lib/activitypub/send/utils.ts +++ b/server/lib/activitypub/send/utils.ts @@ -59,11 +59,11 @@ async function forwardActivity ( async function broadcastToFollowers ( data: any, byActor: ActorModel, - toActorFollowers: ActorModel[], + toFollowersOf: ActorModel[], t: Transaction, actorsException: ActorModel[] = [] ) { - const uris = await computeFollowerUris(toActorFollowers, actorsException, t) + const uris = await computeFollowerUris(toFollowersOf, actorsException, t) return broadcastTo(uris, data, byActor) } @@ -115,8 +115,8 @@ export { // --------------------------------------------------------------------------- -async function computeFollowerUris (toActorFollower: ActorModel[], actorsException: ActorModel[], t: Transaction) { - const toActorFollowerIds = toActorFollower.map(a => a.id) +async function computeFollowerUris (toFollowersOf: ActorModel[], actorsException: ActorModel[], t: Transaction) { + const toActorFollowerIds = toFollowersOf.map(a => a.id) const result = await ActorFollowModel.listAcceptedFollowerSharedInboxUrls(toActorFollowerIds, t) const sharedInboxesException = await buildSharedInboxesException(actorsException) diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts index 262463310..2e7c56955 100644 --- a/server/lib/activitypub/url.ts +++ b/server/lib/activitypub/url.ts @@ -4,11 +4,18 @@ import { ActorFollowModel } from '../../models/activitypub/actor-follow' import { VideoModel } from '../../models/video/video' import { VideoAbuseModel } from '../../models/video/video-abuse' import { VideoCommentModel } from '../../models/video/video-comment' +import { VideoFileModel } from '../../models/video/video-file' function getVideoActivityPubUrl (video: VideoModel) { return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid } +function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) { + const suffixFPS = videoFile.fps ? '-' + videoFile.fps : '' + + return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` +} + function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) { return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id } @@ -101,5 +108,6 @@ export { getVideoSharesActivityPubUrl, getVideoCommentsActivityPubUrl, getVideoLikesActivityPubUrl, - getVideoDislikesActivityPubUrl + getVideoDislikesActivityPubUrl, + getVideoCacheFileActivityPubUrl } diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 6c2095897..783f78d3e 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -3,12 +3,12 @@ import * as sequelize from 'sequelize' import * as magnetUtil from 'magnet-uri' import { join } from 'path' import * as request from 'request' -import { ActivityIconObject, VideoState } from '../../../shared/index' +import { ActivityIconObject, ActivityVideoUrlObject, VideoState, ActivityUrlObject } from '../../../shared/index' import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' import { VideoPrivacy } from '../../../shared/models/videos' import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos' import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' -import { resetSequelizeInstance, retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' +import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers' @@ -17,7 +17,7 @@ import { TagModel } from '../../models/video/tag' import { VideoModel } from '../../models/video/video' import { VideoChannelModel } from '../../models/video/video-channel' import { VideoFileModel } from '../../models/video/video-file' -import { getOrCreateActorAndServerAndModel, updateActorAvatarInstance } from './actor' +import { getOrCreateActorAndServerAndModel } from './actor' import { addVideoComments } from './video-comments' import { crawlCollectionPage } from './crawl' import { sendCreateVideo, sendUpdateVideo } from './send' @@ -25,7 +25,6 @@ import { isArray } from '../../helpers/custom-validators/misc' import { VideoCaptionModel } from '../../models/video/video-caption' import { JobQueue } from '../job-queue' import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher' -import { getUrlFromWebfinger } from '../../helpers/webfinger' import { createRates } from './video-rates' import { addVideoShares, shareVideoByServerAndChannel } from './share' import { AccountModel } from '../../models/account/account' @@ -137,10 +136,7 @@ async function videoActivityObjectToDBAttributes ( } function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) { - const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT) - const fileUrls = videoObject.url.filter(u => { - return mimeTypes.indexOf(u.mimeType) !== -1 && u.mimeType.startsWith('video/') - }) + const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[] if (fileUrls.length === 0) { throw new Error('Cannot find video files for ' + videoCreated.url) @@ -331,8 +327,8 @@ async function refreshVideoIfNeeded (video: VideoModel): Promise { const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject) const account = await AccountModel.load(channelActor.VideoChannel.accountId) - return updateVideoFromAP(video, videoObject, account.Actor, channelActor) + return updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel) } catch (err) { logger.warn('Cannot refresh video.', { err }) return video @@ -342,8 +338,8 @@ async function refreshVideoIfNeeded (video: VideoModel): Promise { async function updateVideoFromAP ( video: VideoModel, videoObject: VideoTorrentObject, - accountActor: ActorModel, - channelActor: ActorModel, + account: AccountModel, + channel: VideoChannelModel, overrideTo?: string[] ) { logger.debug('Updating remote video "%s".', videoObject.uuid) @@ -359,12 +355,12 @@ async function updateVideoFromAP ( // Check actor has the right to update the video const videoChannel = video.VideoChannel - if (videoChannel.Account.Actor.id !== accountActor.id) { - throw new Error('Account ' + accountActor.url + ' does not own video channel ' + videoChannel.Actor.url) + if (videoChannel.Account.id !== account.id) { + throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url) } const to = overrideTo ? overrideTo : videoObject.to - const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, to) + const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, to) video.set('name', videoData.name) video.set('uuid', videoData.uuid) video.set('url', videoData.url) @@ -444,3 +440,11 @@ export { addVideoShares, createRates } + +// --------------------------------------------------------------------------- + +function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject { + const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT) + + return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/') +} diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts new file mode 100644 index 000000000..78221cc3d --- /dev/null +++ b/server/lib/redundancy.ts @@ -0,0 +1,18 @@ +import { VideoRedundancyModel } from '../models/redundancy/video-redundancy' +import { sendUndoCacheFile } from './activitypub/send' +import { Transaction } from 'sequelize' +import { getServerActor } from '../helpers/utils' + +async function removeVideoRedundancy (videoRedundancy: VideoRedundancyModel, t?: Transaction) { + const serverActor = await getServerActor() + + await sendUndoCacheFile(serverActor, videoRedundancy, t) + + await videoRedundancy.destroy({ transaction: t }) +} + +// --------------------------------------------------------------------------- + +export { + removeVideoRedundancy +} diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts new file mode 100644 index 000000000..ee9ba1766 --- /dev/null +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -0,0 +1,161 @@ +import { AbstractScheduler } from './abstract-scheduler' +import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers' +import { logger } from '../../helpers/logger' +import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' +import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' +import { VideoFileModel } from '../../models/video/video-file' +import { sortBy } from 'lodash' +import { downloadWebTorrentVideo } from '../../helpers/webtorrent' +import { join } from 'path' +import { rename } from 'fs-extra' +import { getServerActor } from '../../helpers/utils' +import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' +import { VideoModel } from '../../models/video/video' +import { getVideoCacheFileActivityPubUrl } from '../activitypub/url' +import { removeVideoRedundancy } from '../redundancy' +import { isTestInstance } from '../../helpers/core-utils' + +export class VideosRedundancyScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + private executing = false + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.videosRedundancy + + private constructor () { + super() + } + + async execute () { + if (this.executing) return + + this.executing = true + + for (const obj of CONFIG.REDUNDANCY.VIDEOS) { + + try { + const videoToDuplicate = await this.findVideoToDuplicate(obj.strategy) + if (!videoToDuplicate) continue + + const videoFiles = videoToDuplicate.VideoFiles + videoFiles.forEach(f => f.Video = videoToDuplicate) + + const videosRedundancy = await VideoRedundancyModel.getVideoFiles(obj.strategy) + if (this.isTooHeavy(videosRedundancy, videoFiles, obj.size)) { + if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url) + continue + } + + logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy) + + await this.createVideoRedundancy(obj.strategy, videoFiles) + } catch (err) { + logger.error('Cannot run videos redundancy %s.', obj.strategy, { err }) + } + } + + const expired = await VideoRedundancyModel.listAllExpired() + + for (const m of expired) { + logger.info('Removing expired video %s from our redundancy system.', this.buildEntryLogId(m)) + + try { + await m.destroy() + } catch (err) { + logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m)) + } + } + + this.executing = false + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + private findVideoToDuplicate (strategy: VideoRedundancyStrategy) { + if (strategy === 'most-views') return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) + } + + private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) { + const serverActor = await getServerActor() + + for (const file of filesToDuplicate) { + const existing = await VideoRedundancyModel.loadByFileId(file.id) + if (existing) { + logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', file.Video.url, file.resolution, strategy) + + existing.expiresOn = this.buildNewExpiration() + await existing.save() + + await sendUpdateCacheFile(serverActor, existing) + continue + } + + // We need more attributes and check if the video still exists + const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(file.Video.id) + if (!video) continue + + logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy) + + const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() + const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs) + + const tmpPath = await downloadWebTorrentVideo({ magnetUri }, JOB_TTL['video-import']) + + const destPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file)) + await rename(tmpPath, destPath) + + const createdModel = await VideoRedundancyModel.create({ + expiresOn: new Date(Date.now() + REDUNDANCY.VIDEOS.EXPIRES_AFTER_MS), + url: getVideoCacheFileActivityPubUrl(file), + fileUrl: video.getVideoFileUrl(file, CONFIG.WEBSERVER.URL), + strategy, + videoFileId: file.id, + actorId: serverActor.id + }) + createdModel.VideoFile = file + + await sendCreateCacheFile(serverActor, createdModel) + } + } + + // Unused, but could be useful in the future, with a custom strategy + private async purgeVideosIfNeeded (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSize: number) { + const sortedVideosRedundancy = sortBy(videosRedundancy, 'createdAt') + + while (this.isTooHeavy(sortedVideosRedundancy, filesToDuplicate, maxSize)) { + const toDelete = sortedVideosRedundancy.shift() + + const videoFile = toDelete.VideoFile + logger.info('Purging video %s (resolution %d) from our redundancy system.', videoFile.Video.url, videoFile.resolution) + + await removeVideoRedundancy(toDelete, undefined) + } + + return sortedVideosRedundancy + } + + private isTooHeavy (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSizeArg: number) { + const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate) + + const redundancyReducer = (previous: number, current: VideoRedundancyModel) => previous + current.VideoFile.size + const totalDuplicated = videosRedundancy.reduce(redundancyReducer, 0) + + return totalDuplicated > maxSize + } + + private buildNewExpiration () { + return new Date(Date.now() + REDUNDANCY.VIDEOS.EXPIRES_AFTER_MS) + } + + private buildEntryLogId (object: VideoRedundancyModel) { + return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` + } + + private getTotalFileSizes (files: VideoFileModel[]) { + const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size + + return files.reduce(fileReducer, 0) + } +} diff --git a/server/middlewares/validators/redundancy.ts b/server/middlewares/validators/redundancy.ts new file mode 100644 index 000000000..d91b47574 --- /dev/null +++ b/server/middlewares/validators/redundancy.ts @@ -0,0 +1,80 @@ +import * as express from 'express' +import 'express-validator' +import { param, body } from 'express-validator/check' +import { exists, isBooleanValid, isIdOrUUIDValid, toIntOrNull } from '../../helpers/custom-validators/misc' +import { isVideoExist } from '../../helpers/custom-validators/videos' +import { logger } from '../../helpers/logger' +import { areValidationErrors } from './utils' +import { VideoModel } from '../../models/video/video' +import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' +import { isHostValid } from '../../helpers/custom-validators/servers' +import { getServerActor } from '../../helpers/utils' +import { ActorFollowModel } from '../../models/activitypub/actor-follow' +import { SERVER_ACTOR_NAME } from '../../initializers' +import { ServerModel } from '../../models/server/server' + +const videoRedundancyGetValidator = [ + param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), + param('resolution') + .customSanitizer(toIntOrNull) + .custom(exists).withMessage('Should have a valid resolution'), + param('fps') + .optional() + .customSanitizer(toIntOrNull) + .custom(exists).withMessage('Should have a valid fps'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking videoRedundancyGetValidator parameters', { parameters: req.params }) + + if (areValidationErrors(req, res)) return + if (!await isVideoExist(req.params.videoId, res)) return + + const video: VideoModel = res.locals.video + const videoFile = video.VideoFiles.find(f => { + return f.resolution === req.params.resolution && (!req.params.fps || f.fps === req.params.fps) + }) + + if (!videoFile) return res.status(404).json({ error: 'Video file not found.' }) + res.locals.videoFile = videoFile + + const videoRedundancy = await VideoRedundancyModel.loadByFileId(videoFile.id) + if (!videoRedundancy)return res.status(404).json({ error: 'Video redundancy not found.' }) + res.locals.videoRedundancy = videoRedundancy + + return next() + } +] + +const updateServerRedundancyValidator = [ + param('host').custom(isHostValid).withMessage('Should have a valid host'), + body('redundancyAllowed') + .toBoolean() + .custom(isBooleanValid).withMessage('Should have a valid redundancyAllowed attribute'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking updateServerRedundancy parameters', { parameters: req.params }) + + if (areValidationErrors(req, res)) return + + const server = await ServerModel.loadByHost(req.params.host) + + if (!server) { + return res + .status(404) + .json({ + error: `Server ${req.params.host} not found.` + }) + .end() + } + + res.locals.server = server + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + videoRedundancyGetValidator, + updateServerRedundancyValidator +} diff --git a/server/models/activitypub/actor-follow.ts b/server/models/activitypub/actor-follow.ts index 8bc095997..27bb43dae 100644 --- a/server/models/activitypub/actor-follow.ts +++ b/server/models/activitypub/actor-follow.ts @@ -19,7 +19,7 @@ import { UpdatedAt } from 'sequelize-typescript' import { FollowState } from '../../../shared/models/actors' -import { AccountFollow } from '../../../shared/models/actors/follow.model' +import { ActorFollow } from '../../../shared/models/actors/follow.model' import { logger } from '../../helpers/logger' import { getServerActor } from '../../helpers/utils' import { ACTOR_FOLLOW_SCORE } from '../../initializers' @@ -529,7 +529,7 @@ export class ActorFollowModel extends Model { return ActorFollowModel.findAll(query) } - toFormattedJSON (): AccountFollow { + toFormattedJSON (): ActorFollow { const follower = this.ActorFollower.toFormattedJSON() const following = this.ActorFollowing.toFormattedJSON() diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts index 119d0c1da..ef8dd9f7c 100644 --- a/server/models/activitypub/actor.ts +++ b/server/models/activitypub/actor.ts @@ -76,7 +76,13 @@ export const unusedActorAttributesForAPI = [ }, { model: () => VideoChannelModel.unscoped(), - required: false + required: false, + include: [ + { + model: () => AccountModel, + required: true + } + ] }, { model: () => ServerModel, @@ -337,6 +343,7 @@ export class ActorModel extends Model { uuid: this.uuid, name: this.preferredUsername, host: this.getHost(), + hostRedundancyAllowed: this.getRedundancyAllowed(), followingCount: this.followingCount, followersCount: this.followersCount, avatar, @@ -440,6 +447,10 @@ export class ActorModel extends Model { return this.Server ? this.Server.host : CONFIG.WEBSERVER.HOST } + getRedundancyAllowed () { + return this.Server ? this.Server.redundancyAllowed : false + } + getAvatarUrl () { if (!this.avatarId) return undefined diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts new file mode 100644 index 000000000..48ec77206 --- /dev/null +++ b/server/models/redundancy/video-redundancy.ts @@ -0,0 +1,249 @@ +import { + AfterDestroy, + AllowNull, + BelongsTo, + Column, + CreatedAt, + DataType, + ForeignKey, + Is, + Model, + Scopes, + Sequelize, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { ActorModel } from '../activitypub/actor' +import { throwIfNotValid } from '../utils' +import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc' +import { CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers' +import { VideoFileModel } from '../video/video-file' +import { isDateValid } from '../../helpers/custom-validators/misc' +import { getServerActor } from '../../helpers/utils' +import { VideoModel } from '../video/video' +import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' +import { logger } from '../../helpers/logger' +import { CacheFileObject } from '../../../shared' +import { VideoChannelModel } from '../video/video-channel' +import { ServerModel } from '../server/server' +import { sample } from 'lodash' +import { isTestInstance } from '../../helpers/core-utils' + +export enum ScopeNames { + WITH_VIDEO = 'WITH_VIDEO' +} + +@Scopes({ + [ ScopeNames.WITH_VIDEO ]: { + include: [ + { + model: () => VideoFileModel, + required: true, + include: [ + { + model: () => VideoModel, + required: true + } + ] + } + ] + } +}) + +@Table({ + tableName: 'videoRedundancy', + indexes: [ + { + fields: [ 'videoFileId' ] + }, + { + fields: [ 'actorId' ] + }, + { + fields: [ 'url' ], + unique: true + } + ] +}) +export class VideoRedundancyModel extends Model { + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(false) + @Column + expiresOn: Date + + @AllowNull(false) + @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl')) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max)) + fileUrl: string + + @AllowNull(false) + @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max)) + url: string + + @AllowNull(true) + @Column + strategy: string // Only used by us + + @ForeignKey(() => VideoFileModel) + @Column + videoFileId: number + + @BelongsTo(() => VideoFileModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + VideoFile: VideoFileModel + + @ForeignKey(() => ActorModel) + @Column + actorId: number + + @BelongsTo(() => ActorModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + Actor: ActorModel + + @AfterDestroy + static removeFilesAndSendDelete (instance: VideoRedundancyModel) { + // Not us + if (!instance.strategy) return + + logger.info('Removing video file %s-.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution) + + return instance.VideoFile.Video.removeFile(instance.VideoFile) + } + + static loadByFileId (videoFileId: number) { + const query = { + where: { + videoFileId + } + } + + return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) + } + + static loadByUrl (url: string) { + const query = { + where: { + url + } + } + + return VideoRedundancyModel.findOne(query) + } + + static async findMostViewToDuplicate (randomizedFactor: number) { + // On VideoModel! + const query = { + logging: !isTestInstance(), + limit: randomizedFactor, + order: [ [ 'views', 'DESC' ] ], + include: [ + { + model: VideoFileModel.unscoped(), + required: true, + where: { + id: { + [ Sequelize.Op.notIn ]: await VideoRedundancyModel.buildExcludeIn() + } + } + }, + { + attributes: [], + model: VideoChannelModel.unscoped(), + required: true, + include: [ + { + attributes: [], + model: ActorModel.unscoped(), + required: true, + include: [ + { + attributes: [], + model: ServerModel.unscoped(), + required: true, + where: { + redundancyAllowed: true + } + } + ] + } + ] + } + ] + } + + const rows = await VideoModel.unscoped().findAll(query) + + return sample(rows) + } + + static async getVideoFiles (strategy: VideoRedundancyStrategy) { + const actor = await getServerActor() + + const queryVideoFiles = { + logging: !isTestInstance(), + where: { + actorId: actor.id, + strategy + } + } + + return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO) + .findAll(queryVideoFiles) + } + + static listAllExpired () { + const query = { + logging: !isTestInstance(), + where: { + expiresOn: { + [Sequelize.Op.lt]: new Date() + } + } + } + + return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO) + .findAll(query) + } + + toActivityPubObject (): CacheFileObject { + return { + id: this.url, + type: 'CacheFile' as 'CacheFile', + object: this.VideoFile.Video.url, + expires: this.expiresOn.toISOString(), + url: { + type: 'Link', + mimeType: VIDEO_EXT_MIMETYPE[ this.VideoFile.extname ] as any, + href: this.fileUrl, + height: this.VideoFile.resolution, + size: this.VideoFile.size, + fps: this.VideoFile.fps + } + } + } + + private static async buildExcludeIn () { + const actor = await getServerActor() + + return Sequelize.literal( + '(' + + `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` + + ')' + ) + } +} diff --git a/server/models/server/server.ts b/server/models/server/server.ts index 9749f503e..ca3b24d51 100644 --- a/server/models/server/server.ts +++ b/server/models/server/server.ts @@ -1,4 +1,4 @@ -import { AllowNull, Column, CreatedAt, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { AllowNull, Column, CreatedAt, Default, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' import { isHostValid } from '../../helpers/custom-validators/servers' import { ActorModel } from '../activitypub/actor' import { throwIfNotValid } from '../utils' @@ -19,6 +19,11 @@ export class ServerModel extends Model { @Column host: string + @AllowNull(false) + @Default(false) + @Column + redundancyAllowed: boolean + @CreatedAt createdAt: Date @@ -34,4 +39,14 @@ export class ServerModel extends Model { hooks: true }) Actors: ActorModel[] + + static loadByHost (host: string) { + const query = { + where: { + host + } + } + + return ServerModel.findOne(query) + } } diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 3bc4855f3..0907ea569 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -1,5 +1,18 @@ import { values } from 'lodash' -import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { + AllowNull, + BelongsTo, + Column, + CreatedAt, + DataType, + Default, + ForeignKey, + HasMany, + Is, + Model, + Table, + UpdatedAt +} from 'sequelize-typescript' import { isVideoFileInfoHashValid, isVideoFileResolutionValid, @@ -10,6 +23,7 @@ import { CONSTRAINTS_FIELDS } from '../../initializers' import { throwIfNotValid } from '../utils' import { VideoModel } from './video' import * as Sequelize from 'sequelize' +import { VideoRedundancyModel } from '../redundancy/video-redundancy' @Table({ tableName: 'videoFile', @@ -70,6 +84,15 @@ export class VideoFileModel extends Model { }) Video: VideoModel + @HasMany(() => VideoRedundancyModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE', + hooks: true + }) + RedundancyVideos: VideoRedundancyModel[] + static isInfohashExists (infoHash: string) { const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' const options = { diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 86316653f..27c631dcd 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -27,13 +27,13 @@ import { Table, UpdatedAt } from 'sequelize-typescript' -import { VideoPrivacy, VideoResolution, VideoState } from '../../../shared' +import { ActivityUrlObject, VideoPrivacy, VideoResolution, VideoState } from '../../../shared' import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' import { VideoFilter } from '../../../shared/models/videos/video-query.type' import { createTorrentPromise, peertubeTruncate } from '../../helpers/core-utils' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { isBooleanValid } from '../../helpers/custom-validators/misc' +import { isArray, isBooleanValid } from '../../helpers/custom-validators/misc' import { isVideoCategoryValid, isVideoDescriptionValid, @@ -90,6 +90,7 @@ import { VideoCaptionModel } from './video-caption' import { VideoBlacklistModel } from './video-blacklist' import { copy, remove, rename, stat, writeFile } from 'fs-extra' import { VideoViewModel } from './video-views' +import { VideoRedundancyModel } from '../redundancy/video-redundancy' // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation const indexes: Sequelize.DefineIndexesOptions[] = [ @@ -470,7 +471,13 @@ type AvailableForListIDsOptions = { include: [ { model: () => VideoFileModel.unscoped(), - required: false + required: false, + include: [ + { + model: () => VideoRedundancyModel.unscoped(), + required: false + } + ] } ] }, @@ -633,6 +640,7 @@ export class VideoModel extends Model { name: 'videoId', allowNull: false }, + hooks: true, onDelete: 'cascade' }) VideoFiles: VideoFileModel[] @@ -1325,9 +1333,7 @@ export class VideoModel extends Model { [ CONFIG.WEBSERVER.WS + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT + '/tracker/socket' ], [ CONFIG.WEBSERVER.URL + '/tracker/announce' ] ], - urlList: [ - CONFIG.WEBSERVER.URL + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) - ] + urlList: [ CONFIG.WEBSERVER.URL + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) ] } const torrent = await createTorrentPromise(this.getVideoFilePath(videoFile), options) @@ -1535,11 +1541,11 @@ export class VideoModel extends Model { } } - const url = [] + const url: ActivityUrlObject[] = [] for (const file of this.VideoFiles) { url.push({ type: 'Link', - mimeType: VIDEO_EXT_MIMETYPE[ file.extname ], + mimeType: VIDEO_EXT_MIMETYPE[ file.extname ] as any, href: this.getVideoFileUrl(file, baseUrlHttp), height: file.resolution, size: file.size, @@ -1548,14 +1554,14 @@ export class VideoModel extends Model { url.push({ type: 'Link', - mimeType: 'application/x-bittorrent', + mimeType: 'application/x-bittorrent' as 'application/x-bittorrent', href: this.getTorrentUrl(file, baseUrlHttp), height: file.resolution }) url.push({ type: 'Link', - mimeType: 'application/x-bittorrent;x-scheme-handler/magnet', + mimeType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet', href: this.generateMagnetUri(file, baseUrlHttp, baseUrlWs), height: file.resolution }) @@ -1796,7 +1802,7 @@ export class VideoModel extends Model { (now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL } - private getBaseUrls () { + getBaseUrls () { let baseUrlHttp let baseUrlWs @@ -1811,39 +1817,42 @@ export class VideoModel extends Model { return { baseUrlHttp, baseUrlWs } } - private getThumbnailUrl (baseUrlHttp: string) { + generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) { + const xs = this.getTorrentUrl(videoFile, baseUrlHttp) + const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] + let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ] + + const redundancies = videoFile.RedundancyVideos + if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl)) + + const magnetHash = { + xs, + announce, + urlList, + infoHash: videoFile.infoHash, + name: this.name + } + + return magnetUtil.encode(magnetHash) + } + + getThumbnailUrl (baseUrlHttp: string) { return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName() } - private getTorrentUrl (videoFile: VideoFileModel, baseUrlHttp: string) { + getTorrentUrl (videoFile: VideoFileModel, baseUrlHttp: string) { return baseUrlHttp + STATIC_PATHS.TORRENTS + this.getTorrentFileName(videoFile) } - private getTorrentDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { + getTorrentDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { return baseUrlHttp + STATIC_DOWNLOAD_PATHS.TORRENTS + this.getTorrentFileName(videoFile) } - private getVideoFileUrl (videoFile: VideoFileModel, baseUrlHttp: string) { + getVideoFileUrl (videoFile: VideoFileModel, baseUrlHttp: string) { return baseUrlHttp + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) } - private getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { + getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) { return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile) } - - private generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) { - const xs = this.getTorrentUrl(videoFile, baseUrlHttp) - const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] - const urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ] - - const magnetHash = { - xs, - announce, - urlList, - infoHash: videoFile.infoHash, - name: this.name - } - - return magnetUtil.encode(magnetHash) - } } diff --git a/server/tests/api/check-params/follows.ts b/server/tests/api/check-params/follows.ts index 2bc3b27d9..cdc95c81a 100644 --- a/server/tests/api/check-params/follows.ts +++ b/server/tests/api/check-params/follows.ts @@ -169,15 +169,6 @@ describe('Test server follows API validators', function () { statusCodeExpected: 404 }) }) - - it('Should succeed with the correct parameters', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/localhost:9002', - token: server.accessToken, - statusCodeExpected: 404 - }) - }) }) }) diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts index 777acbb0f..44460a167 100644 --- a/server/tests/api/check-params/index.ts +++ b/server/tests/api/check-params/index.ts @@ -1,15 +1,17 @@ // Order of the tests we want to execute import './accounts' +import './config' import './follows' import './jobs' +import './redundancy' +import './search' import './services' +import './user-subscriptions' import './users' import './video-abuses' import './video-blacklist' import './video-captions' import './video-channels' import './video-comments' -import './videos' import './video-imports' -import './search' -import './user-subscriptions' +import './videos' diff --git a/server/tests/api/check-params/redundancy.ts b/server/tests/api/check-params/redundancy.ts new file mode 100644 index 000000000..aa588e3dd --- /dev/null +++ b/server/tests/api/check-params/redundancy.ts @@ -0,0 +1,103 @@ +/* tslint:disable:no-unused-expression */ + +import 'mocha' + +import { + createUser, + doubleFollow, + flushAndRunMultipleServers, + flushTests, + killallServers, + makePutBodyRequest, + ServerInfo, + setAccessTokensToServers, + userLogin +} from '../../utils' + +describe('Test server redundancy API validators', function () { + let servers: ServerInfo[] + let userAccessToken = null + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + await flushTests() + servers = await flushAndRunMultipleServers(2) + + await setAccessTokensToServers(servers) + await doubleFollow(servers[0], servers[1]) + + const user = { + username: 'user1', + password: 'password' + } + + await createUser(servers[0].url, servers[0].accessToken, user.username, user.password) + userAccessToken = await userLogin(servers[0], user) + }) + + describe('When updating redundancy', function () { + const path = '/api/v1/server/redundancy' + + it('Should fail with an invalid token', async function () { + await makePutBodyRequest({ + url: servers[0].url, + path: path + '/localhost:9002', + fields: { redundancyAllowed: true }, + token: 'fake_token', + statusCodeExpected: 401 + }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makePutBodyRequest({ + url: servers[0].url, + path: path + '/localhost:9002', + fields: { redundancyAllowed: true }, + token: userAccessToken, + statusCodeExpected: 403 + }) + }) + + it('Should fail if we do not follow this server', async function () { + await makePutBodyRequest({ + url: servers[0].url, + path: path + '/example.com', + fields: { redundancyAllowed: true }, + token: servers[0].accessToken, + statusCodeExpected: 404 + }) + }) + + it('Should fail without de redundancyAllowed param', async function () { + await makePutBodyRequest({ + url: servers[0].url, + path: path + '/localhost:9002', + fields: { blabla: true }, + token: servers[0].accessToken, + statusCodeExpected: 400 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await makePutBodyRequest({ + url: servers[0].url, + path: path + '/localhost:9002', + fields: { redundancyAllowed: true }, + token: servers[0].accessToken, + statusCodeExpected: 204 + }) + }) + }) + + after(async function () { + killallServers(servers) + + // Keep the logs if the test failed + if (this['ok']) { + await flushTests() + } + }) +}) diff --git a/server/tests/api/server/index.ts b/server/tests/api/server/index.ts index eeb8b7a28..c74c68a33 100644 --- a/server/tests/api/server/index.ts +++ b/server/tests/api/server/index.ts @@ -3,6 +3,7 @@ import './email' import './follows' import './handle-down' import './jobs' +import './redundancy' import './reverse-proxy' import './stats' import './tracker' diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts new file mode 100644 index 000000000..c0ec75a45 --- /dev/null +++ b/server/tests/api/server/redundancy.ts @@ -0,0 +1,140 @@ +/* tslint:disable:no-unused-expression */ + +import * as chai from 'chai' +import 'mocha' +import { VideoDetails } from '../../../../shared/models/videos' +import { + doubleFollow, + flushAndRunMultipleServers, + flushTests, + getFollowingListPaginationAndSort, + getVideo, + killallServers, + ServerInfo, + setAccessTokensToServers, + uploadVideo, + wait, + root, viewVideo +} from '../../utils' +import { waitJobs } from '../../utils/server/jobs' +import * as magnetUtil from 'magnet-uri' +import { updateRedundancy } from '../../utils/server/redundancy' +import { ActorFollow } from '../../../../shared/models/actors' +import { readdir } from 'fs-extra' +import { join } from 'path' + +const expect = chai.expect + +function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) { + const parsed = magnetUtil.decode(file.magnetUri) + + for (const ws of baseWebseeds) { + const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`) + expect(found, `Webseed ${ws} not found in ${file.magnetUri}`).to.not.be.undefined + } +} + +describe('Test videos redundancy', function () { + let servers: ServerInfo[] = [] + let video1Server2UUID: string + let video2Server2UUID: string + + before(async function () { + this.timeout(120000) + + servers = await flushAndRunMultipleServers(3) + + // Get the access tokens + await setAccessTokensToServers(servers) + + { + const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) + video1Server2UUID = res.body.video.uuid + + await viewVideo(servers[1].url, video1Server2UUID) + } + + { + const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) + video2Server2UUID = res.body.video.uuid + } + + await waitJobs(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + // Server 1 and server 3 follow each other + await doubleFollow(servers[0], servers[2]) + // Server 2 and server 3 follow each other + await doubleFollow(servers[1], servers[2]) + + await waitJobs(servers) + }) + + it('Should have 1 webseed on the first video', async function () { + const webseeds = [ + 'http://localhost:9002/static/webseed/' + video1Server2UUID + ] + + for (const server of servers) { + const res = await getVideo(server.url, video1Server2UUID) + + const video: VideoDetails = res.body + video.files.forEach(f => checkMagnetWebseeds(f, webseeds)) + } + }) + + it('Should enable redundancy on server 1', async function () { + await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true) + + const res = await getFollowingListPaginationAndSort(servers[0].url, 0, 5, '-createdAt') + const follows: ActorFollow[] = res.body.data + const server2 = follows.find(f => f.following.host === 'localhost:9002') + const server3 = follows.find(f => f.following.host === 'localhost:9003') + + expect(server3).to.not.be.undefined + expect(server3.following.hostRedundancyAllowed).to.be.false + + expect(server2).to.not.be.undefined + expect(server2.following.hostRedundancyAllowed).to.be.true + }) + + it('Should have 2 webseed on the first video', async function () { + this.timeout(40000) + + await waitJobs(servers) + await wait(15000) + await waitJobs(servers) + + const webseeds = [ + 'http://localhost:9001/static/webseed/' + video1Server2UUID, + 'http://localhost:9002/static/webseed/' + video1Server2UUID + ] + + for (const server of servers) { + const res = await getVideo(server.url, video1Server2UUID) + + const video: VideoDetails = res.body + + for (const file of video.files) { + checkMagnetWebseeds(file, webseeds) + } + } + + const files = await readdir(join(root(), 'test1', 'videos')) + expect(files).to.have.lengthOf(4) + + for (const resolution of [ 240, 360, 480, 720 ]) { + expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined + } + }) + + after(async function () { + killallServers(servers) + + // Keep the logs if the test failed + if (this['ok']) { + await flushTests() + } + }) +}) diff --git a/server/tests/utils/server/follows.ts b/server/tests/utils/server/follows.ts index d21fb5e58..8a65a958b 100644 --- a/server/tests/utils/server/follows.ts +++ b/server/tests/utils/server/follows.ts @@ -1,5 +1,4 @@ import * as request from 'supertest' -import { wait } from '../miscs/miscs' import { ServerInfo } from './servers' import { waitJobs } from './jobs' diff --git a/server/tests/utils/server/redundancy.ts b/server/tests/utils/server/redundancy.ts new file mode 100644 index 000000000..c39ff2c8b --- /dev/null +++ b/server/tests/utils/server/redundancy.ts @@ -0,0 +1,17 @@ +import { makePutBodyRequest } from '../requests/requests' + +async function updateRedundancy (url: string, accessToken: string, host: string, redundancyAllowed: boolean, expectedStatus = 204) { + const path = '/api/v1/server/redundancy/' + host + + return makePutBodyRequest({ + url, + path, + token: accessToken, + fields: { redundancyAllowed }, + statusCodeExpected: expectedStatus + }) +} + +export { + updateRedundancy +} diff --git a/shared/models/activitypub/activity.ts b/shared/models/activitypub/activity.ts index 46e883e5f..44cb99efb 100644 --- a/shared/models/activitypub/activity.ts +++ b/shared/models/activitypub/activity.ts @@ -1,6 +1,6 @@ import { ActivityPubActor } from './activitypub-actor' import { ActivityPubSignature } from './activitypub-signature' -import { VideoTorrentObject } from './objects' +import { CacheFileObject, VideoTorrentObject } from './objects' import { DislikeObject } from './objects/dislike-object' import { VideoAbuseObject } from './objects/video-abuse-object' import { VideoCommentObject } from './objects/video-comment-object' @@ -29,12 +29,12 @@ export interface BaseActivity { export interface ActivityCreate extends BaseActivity { type: 'Create' - object: VideoTorrentObject | VideoAbuseObject | ViewObject | DislikeObject | VideoCommentObject + object: VideoTorrentObject | VideoAbuseObject | ViewObject | DislikeObject | VideoCommentObject | CacheFileObject } export interface ActivityUpdate extends BaseActivity { type: 'Update' - object: VideoTorrentObject | ActivityPubActor + object: VideoTorrentObject | ActivityPubActor | CacheFileObject } export interface ActivityDelete extends BaseActivity { diff --git a/shared/models/activitypub/objects/cache-file-object.ts b/shared/models/activitypub/objects/cache-file-object.ts new file mode 100644 index 000000000..0a5125f5b --- /dev/null +++ b/shared/models/activitypub/objects/cache-file-object.ts @@ -0,0 +1,9 @@ +import { ActivityVideoUrlObject } from './common-objects' + +export interface CacheFileObject { + id: string + type: 'CacheFile', + object: string + expires: string + url: ActivityVideoUrlObject +} diff --git a/shared/models/activitypub/objects/common-objects.ts b/shared/models/activitypub/objects/common-objects.ts index ff2cfdbb4..1de60da94 100644 --- a/shared/models/activitypub/objects/common-objects.ts +++ b/shared/models/activitypub/objects/common-objects.ts @@ -17,16 +17,31 @@ export interface ActivityIconObject { height: number } -export interface ActivityUrlObject { +export type ActivityVideoUrlObject = { type: 'Link' - mimeType: 'video/mp4' | 'video/webm' | 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet' + mimeType: 'video/mp4' | 'video/webm' | 'video/ogg' href: string height: number - - size?: number - fps?: number + size: number + fps: number } +export type ActivityUrlObject = + ActivityVideoUrlObject + | + { + type: 'Link' + mimeType: 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet' + href: string + height: number + } + | + { + type: 'Link' + mimeType: 'text/html' + href: string + } + export interface ActivityPubAttributedTo { type: 'Group' | 'Person' id: string diff --git a/shared/models/activitypub/objects/index.ts b/shared/models/activitypub/objects/index.ts index 3efd3ef13..fba61e12f 100644 --- a/shared/models/activitypub/objects/index.ts +++ b/shared/models/activitypub/objects/index.ts @@ -1,3 +1,4 @@ +export * from './cache-file-object' export * from './common-objects' export * from './video-abuse-object' export * from './video-torrent-object' diff --git a/shared/models/activitypub/objects/video-torrent-object.ts b/shared/models/activitypub/objects/video-torrent-object.ts index 90de8967b..8504c178f 100644 --- a/shared/models/activitypub/objects/video-torrent-object.ts +++ b/shared/models/activitypub/objects/video-torrent-object.ts @@ -1,10 +1,10 @@ import { ActivityIconObject, - ActivityIdentifierObject, ActivityPubAttributedTo, + ActivityIdentifierObject, + ActivityPubAttributedTo, ActivityTagObject, ActivityUrlObject } from './common-objects' -import { ActivityPubOrderedCollection } from '../activitypub-ordered-collection' import { VideoState } from '../../videos' export interface VideoTorrentObject { diff --git a/shared/models/actors/follow.model.ts b/shared/models/actors/follow.model.ts index 70562bfc7..7de638cba 100644 --- a/shared/models/actors/follow.model.ts +++ b/shared/models/actors/follow.model.ts @@ -2,10 +2,10 @@ import { Actor } from './actor.model' export type FollowState = 'pending' | 'accepted' -export interface AccountFollow { +export interface ActorFollow { id: number - follower: Actor - following: Actor + follower: Actor & { hostRedundancyAllowed: boolean } + following: Actor & { hostRedundancyAllowed: boolean } score: number state: FollowState createdAt: Date diff --git a/shared/models/avatars/index.ts b/shared/models/avatars/index.ts new file mode 100644 index 000000000..65e8e0882 --- /dev/null +++ b/shared/models/avatars/index.ts @@ -0,0 +1 @@ +export * from './avatar.model' diff --git a/shared/models/index.ts b/shared/models/index.ts index 170f620e7..e61d6cbdc 100644 --- a/shared/models/index.ts +++ b/shared/models/index.ts @@ -1,5 +1,7 @@ -export * from './actors' export * from './activitypub' +export * from './actors' +export * from './avatars' +export * from './redundancy' export * from './users' export * from './videos' export * from './feeds' diff --git a/shared/models/redundancy/index.ts b/shared/models/redundancy/index.ts new file mode 100644 index 000000000..61bf0fca7 --- /dev/null +++ b/shared/models/redundancy/index.ts @@ -0,0 +1 @@ +export * from './videos-redundancy.model' diff --git a/shared/models/redundancy/videos-redundancy.model.ts b/shared/models/redundancy/videos-redundancy.model.ts new file mode 100644 index 000000000..eb84964e0 --- /dev/null +++ b/shared/models/redundancy/videos-redundancy.model.ts @@ -0,0 +1,6 @@ +export type VideoRedundancyStrategy = 'most-views' + +export interface VideosRedundancy { + strategy: VideoRedundancyStrategy + size: number +} diff --git a/shared/models/users/user-right.enum.ts b/shared/models/users/user-right.enum.ts index 64ad3e9b9..c4ccd632f 100644 --- a/shared/models/users/user-right.enum.ts +++ b/shared/models/users/user-right.enum.ts @@ -3,6 +3,7 @@ export enum UserRight { MANAGE_USERS, MANAGE_SERVER_FOLLOW, + MANAGE_SERVER_REDUNDANCY, MANAGE_VIDEO_ABUSES, MANAGE_JOBS, MANAGE_CONFIGURATION, diff --git a/yarn.lock b/yarn.lock index 735d56715..c8fb21117 100644 --- a/yarn.lock +++ b/yarn.lock @@ -44,6 +44,10 @@ "@types/bluebird" "*" "@types/ioredis" "*" +"@types/bytes@^3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@types/bytes/-/bytes-3.0.0.tgz#549eeacd0a8fecfaa459334583a4edcee738e6db" + "@types/caseless@*": version "0.12.1" resolved "https://registry.yarnpkg.com/@types/caseless/-/caseless-0.12.1.tgz#9794c69c8385d0192acc471a540d1f8e0d16218a" @@ -993,7 +997,7 @@ bytes@1: version "1.0.0" resolved "https://registry.yarnpkg.com/bytes/-/bytes-1.0.0.tgz#3569ede8ba34315fab99c3e92cb04c7220de1fa8" -bytes@3.0.0: +bytes@3.0.0, bytes@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.0.0.tgz#d32815404d689699f85a4ea4fa8755dd13a96048"