import { ModerationCommentModalComponent, VideoAbuseListComponent, VideoBlacklistListComponent } from './moderation'
import { UserBanModalComponent } from '@app/+admin/users/user-list/user-ban-modal.component'
import { ModerationComponent } from '@app/+admin/moderation/moderation.component'
+import { RedundancyCheckboxComponent } from '@app/+admin/follows/shared/redundancy-checkbox.component'
+import { RedundancyService } from '@app/+admin/follows/shared/redundancy.service'
@NgModule({
imports: [
FollowingAddComponent,
FollowersListComponent,
FollowingListComponent,
+ RedundancyCheckboxComponent,
UsersComponent,
UserCreateComponent,
providers: [
FollowService,
+ RedundancyService,
UserService,
JobService,
ConfigService
import { NotificationsService } from 'angular2-notifications'
import { SortMeta } from 'primeng/primeng'
-import { AccountFollow } from '../../../../../../shared/models/actors/follow.model'
+import { ActorFollow } from '../../../../../../shared/models/actors/follow.model'
import { RestPagination, RestTable } from '../../../shared'
import { FollowService } from '../shared'
import { I18n } from '@ngx-translate/i18n-polyfill'
styleUrls: [ './followers-list.component.scss' ]
})
export class FollowersListComponent extends RestTable implements OnInit {
- followers: AccountFollow[] = []
+ followers: ActorFollow[] = []
totalRecords = 0
rowsPerPage = 10
sort: SortMeta = { field: 'createdAt', order: 1 }
<th i18n>Host</th>
<th i18n>State</th>
<th i18n pSortableColumn="createdAt">Created <p-sortIcon field="createdAt"></p-sortIcon></th>
+ <th i18n>Redundancy allowed</th>
<th></th>
</tr>
</ng-template>
<td>{{ follow.following.host }}</td>
<td>{{ follow.state }}</td>
<td>{{ follow.createdAt }}</td>
+ <td>
+ <my-redundancy-checkbox
+ [host]="follow.following.host" [redundancyAllowed]="follow.following.hostRedundancyAllowed"
+ ></my-redundancy-checkbox>
+ </td>
<td class="action-cell">
<my-delete-button (click)="removeFollowing(follow)"></my-delete-button>
</td>
--- /dev/null
+@import '_variables';
+@import '_mixins';
+
+my-redundancy-checkbox /deep/ my-peertube-checkbox {
+ .form-group {
+ margin-bottom: 0;
+ align-items: center;
+ }
+
+ label {
+ margin: 0;
+ }
+}
\ No newline at end of file
import { Component, OnInit } from '@angular/core'
import { NotificationsService } from 'angular2-notifications'
import { SortMeta } from 'primeng/primeng'
-import { AccountFollow } from '../../../../../../shared/models/actors/follow.model'
+import { ActorFollow } from '../../../../../../shared/models/actors/follow.model'
import { ConfirmService } from '../../../core/confirm/confirm.service'
import { RestPagination, RestTable } from '../../../shared'
import { FollowService } from '../shared'
@Component({
selector: 'my-followers-list',
- templateUrl: './following-list.component.html'
+ templateUrl: './following-list.component.html',
+ styleUrls: [ './following-list.component.scss' ]
})
export class FollowingListComponent extends RestTable implements OnInit {
- following: AccountFollow[] = []
+ following: ActorFollow[] = []
totalRecords = 0
rowsPerPage = 10
sort: SortMeta = { field: 'createdAt', order: 1 }
this.loadSort()
}
- async removeFollowing (follow: AccountFollow) {
+ async removeFollowing (follow: ActorFollow) {
const res = await this.confirmService.confirm(
this.i18n('Do you really want to unfollow {{host}}?', { host: follow.following.host }),
this.i18n('Unfollow')
import { Injectable } from '@angular/core'
import { SortMeta } from 'primeng/primeng'
import { Observable } from 'rxjs'
-import { AccountFollow, ResultList } from '../../../../../../shared'
+import { ActorFollow, ResultList } from '../../../../../../shared'
import { environment } from '../../../../environments/environment'
import { RestExtractor, RestPagination, RestService } from '../../../shared'
) {
}
- getFollowing (pagination: RestPagination, sort: SortMeta): Observable<ResultList<AccountFollow>> {
+ getFollowing (pagination: RestPagination, sort: SortMeta): Observable<ResultList<ActorFollow>> {
let params = new HttpParams()
params = this.restService.addRestGetParams(params, pagination, sort)
- return this.authHttp.get<ResultList<AccountFollow>>(FollowService.BASE_APPLICATION_URL + '/following', { params })
+ return this.authHttp.get<ResultList<ActorFollow>>(FollowService.BASE_APPLICATION_URL + '/following', { params })
.pipe(
map(res => this.restExtractor.convertResultListDateToHuman(res)),
catchError(res => this.restExtractor.handleError(res))
)
}
- getFollowers (pagination: RestPagination, sort: SortMeta): Observable<ResultList<AccountFollow>> {
+ getFollowers (pagination: RestPagination, sort: SortMeta): Observable<ResultList<ActorFollow>> {
let params = new HttpParams()
params = this.restService.addRestGetParams(params, pagination, sort)
- return this.authHttp.get<ResultList<AccountFollow>>(FollowService.BASE_APPLICATION_URL + '/followers', { params })
+ return this.authHttp.get<ResultList<ActorFollow>>(FollowService.BASE_APPLICATION_URL + '/followers', { params })
.pipe(
map(res => this.restExtractor.convertResultListDateToHuman(res)),
catchError(res => this.restExtractor.handleError(res))
)
}
- unfollow (follow: AccountFollow) {
+ unfollow (follow: ActorFollow) {
return this.authHttp.delete(FollowService.BASE_APPLICATION_URL + '/following/' + follow.following.host)
.pipe(
map(this.restExtractor.extractDataBool),
--- /dev/null
+<my-peertube-checkbox
+ [inputName]="host + '-redundancy-allowed'" [(ngModel)]="redundancyAllowed" (ngModelChange)="updateRedundancyState()"
+></my-peertube-checkbox>
\ No newline at end of file
--- /dev/null
+@import '_variables';
+@import '_mixins';
--- /dev/null
+import { Component, Input } from '@angular/core'
+import { AuthService } from '@app/core'
+import { RestExtractor } from '@app/shared/rest'
+import { RedirectService } from '@app/core/routing/redirect.service'
+import { NotificationsService } from 'angular2-notifications'
+import { I18n } from '@ngx-translate/i18n-polyfill'
+import { RedundancyService } from '@app/+admin/follows/shared/redundancy.service'
+
+@Component({
+ selector: 'my-redundancy-checkbox',
+ templateUrl: './redundancy-checkbox.component.html',
+ styleUrls: [ './redundancy-checkbox.component.scss' ]
+})
+export class RedundancyCheckboxComponent {
+ @Input() redundancyAllowed: boolean
+ @Input() host: string
+
+ constructor (
+ private authService: AuthService,
+ private restExtractor: RestExtractor,
+ private redirectService: RedirectService,
+ private notificationsService: NotificationsService,
+ private redundancyService: RedundancyService,
+ private i18n: I18n
+ ) { }
+
+ updateRedundancyState () {
+ this.redundancyService.updateRedundancy(this.host, this.redundancyAllowed)
+ .subscribe(
+ () => {
+ const stateLabel = this.redundancyAllowed ? this.i18n('enabled') : this.i18n('disabled')
+
+ this.notificationsService.success(
+ this.i18n('Success'),
+ this.i18n('Redundancy for {{host}} is {{stateLabel}}', { host: this.host, stateLabel })
+ )
+ },
+
+ err => this.notificationsService.error(this.i18n('Error'), err.message)
+ )
+ }
+}
--- /dev/null
+import { catchError, map } from 'rxjs/operators'
+import { HttpClient } from '@angular/common/http'
+import { Injectable } from '@angular/core'
+import { RestExtractor, RestService } from '@app/shared'
+import { environment } from '../../../../environments/environment'
+
+@Injectable()
+export class RedundancyService {
+ static BASE_USER_SUBSCRIPTIONS_URL = environment.apiUrl + '/api/v1/server/redundancy'
+
+ constructor (
+ private authHttp: HttpClient,
+ private restExtractor: RestExtractor,
+ private restService: RestService
+ ) { }
+
+ updateRedundancy (host: string, redundancyAllowed: boolean) {
+ const url = RedundancyService.BASE_USER_SUBSCRIPTIONS_URL + '/' + host
+
+ const body = { redundancyAllowed }
+
+ return this.authHttp.put(url, body)
+ .pipe(
+ map(this.restExtractor.extractDataBool),
+ catchError(err => this.restExtractor.handleError(err))
+ )
+ }
+
+}
videos:
interval_days: 7 # Compute trending videos for the last x days
+# Cache remote videos on your server, to help other instances to broadcast the video
+# You can define multiple caches using different sizes/strategies
+# Once you have defined your strategies, choose which instances you want to cache in admin -> manage follows -> following
+redundancy:
+ videos:
+# -
+# size: '10GB'
+# strategy: 'most-views' # Cache videos that have the most views
+
cache:
previews:
size: 500 # Max number of previews you want to cache
videos:
interval_days: 7 # Compute trending videos for the last x days
+# Cache remote videos on your server, to help other instances to broadcast the video
+# You can define multiple caches using different sizes/strategies
+# Once you have defined your strategies, choose which instances you want to cache in admin -> manage follows -> following
+redundancy:
+ videos:
+# -
+# size: '10GB'
+# strategy: 'most-views' # Cache videos that have the most views
+
###############################################################################
#
# From this point, all the following keys can be overridden by the web interface
log:
level: 'debug'
+redundancy:
+ videos:
+ -
+ size: '100KB'
+ strategy: 'most-views'
+
cache:
previews:
size: 1
"bluebird": "^3.5.0",
"body-parser": "^1.12.4",
"bull": "^3.4.2",
+ "bytes": "^3.0.0",
"commander": "^2.13.0",
"concurrently": "^4.0.1",
"config": "^2.0.1",
"@types/bluebird": "3.5.21",
"@types/body-parser": "^1.16.3",
"@types/bull": "^3.3.12",
+ "@types/bytes": "^3.0.0",
"@types/chai": "^4.0.4",
"@types/chai-json-schema": "^1.4.3",
"@types/chai-xml": "^0.3.1",
import { RemoveOldJobsScheduler } from './server/lib/schedulers/remove-old-jobs-scheduler'
import { UpdateVideosScheduler } from './server/lib/schedulers/update-videos-scheduler'
import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler'
+import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler'
// ----------- Command line -----------
RemoveOldJobsScheduler.Instance.enable()
UpdateVideosScheduler.Instance.enable()
YoutubeDlUpdateScheduler.Instance.enable()
+ VideosRedundancyScheduler.Instance.enable()
// Redis initialization
Redis.Instance.init()
import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos'
import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub'
import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../initializers'
-import { buildVideoAnnounce } from '../../lib/activitypub/send'
+import { buildAnnounceWithVideoAudience } from '../../lib/activitypub/send'
import { audiencify, getAudience } from '../../lib/activitypub/audience'
-import { createActivityData } from '../../lib/activitypub/send/send-create'
+import { buildCreateActivity } from '../../lib/activitypub/send/send-create'
import { asyncMiddleware, executeIfActivityPub, localAccountValidator, localVideoChannelValidator } from '../../middlewares'
import { videosGetValidator, videosShareValidator } from '../../middlewares/validators'
import { videoCommentGetValidator } from '../../middlewares/validators/video-comments'
getVideoSharesActivityPubUrl
} from '../../lib/activitypub'
import { VideoCaptionModel } from '../../models/video/video-caption'
+import { videoRedundancyGetValidator } from '../../middlewares/validators/redundancy'
+import { getServerActor } from '../../helpers/utils'
const activityPubClientRouter = express.Router()
executeIfActivityPub(asyncMiddleware(videoChannelFollowingController))
)
+activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?',
+ executeIfActivityPub(asyncMiddleware(videoRedundancyGetValidator)),
+ executeIfActivityPub(asyncMiddleware(videoRedundancyController))
+)
+
// ---------------------------------------------------------------------------
export {
const videoObject = audiencify(video.toActivityPubObject(), audience)
if (req.path.endsWith('/activity')) {
- const data = createActivityData(video.url, video.VideoChannel.Account.Actor, videoObject, audience)
+ const data = buildCreateActivity(video.url, video.VideoChannel.Account.Actor, videoObject, audience)
return activityPubResponse(activityPubContextify(data), res)
}
async function videoAnnounceController (req: express.Request, res: express.Response, next: express.NextFunction) {
const share = res.locals.videoShare as VideoShareModel
- const object = await buildVideoAnnounce(share.Actor, share, res.locals.video, undefined)
+ const { activity } = await buildAnnounceWithVideoAudience(share.Actor, share, res.locals.video, undefined)
- return activityPubResponse(activityPubContextify(object), res)
+ return activityPubResponse(activityPubContextify(activity), res)
}
async function videoAnnouncesController (req: express.Request, res: express.Response, next: express.NextFunction) {
const videoCommentObject = audiencify(videoComment.toActivityPubObject(threadParentComments), audience)
if (req.path.endsWith('/activity')) {
- const data = createActivityData(videoComment.url, videoComment.Account.Actor, videoCommentObject, audience)
+ const data = buildCreateActivity(videoComment.url, videoComment.Account.Actor, videoCommentObject, audience)
return activityPubResponse(activityPubContextify(data), res)
}
return activityPubResponse(activityPubContextify(videoCommentObject), res)
}
+async function videoRedundancyController (req: express.Request, res: express.Response) {
+ const videoRedundancy = res.locals.videoRedundancy
+ const serverActor = await getServerActor()
+
+ const audience = getAudience(serverActor)
+ const object = audiencify(videoRedundancy.toActivityPubObject(), audience)
+
+ if (req.path.endsWith('/activity')) {
+ const data = buildCreateActivity(videoRedundancy.url, serverActor, object, audience)
+ return activityPubResponse(activityPubContextify(data), res)
+ }
+
+ return activityPubResponse(activityPubContextify(object), res)
+}
+
// ---------------------------------------------------------------------------
async function actorFollowing (req: express.Request, actor: ActorModel) {
import { VideoPrivacy } from '../../../shared/models/videos'
import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub'
import { logger } from '../../helpers/logger'
-import { announceActivityData, createActivityData } from '../../lib/activitypub/send'
+import { buildAnnounceActivity, buildCreateActivity } from '../../lib/activitypub/send'
import { buildAudience } from '../../lib/activitypub/audience'
import { asyncMiddleware, localAccountValidator, localVideoChannelValidator } from '../../middlewares'
import { AccountModel } from '../../models/account/account'
// This is a shared video
if (video.VideoShares !== undefined && video.VideoShares.length !== 0) {
const videoShare = video.VideoShares[0]
- const announceActivity = announceActivityData(videoShare.url, actor, video.url, createActivityAudience)
+ const announceActivity = buildAnnounceActivity(videoShare.url, actor, video.url, createActivityAudience)
activities.push(announceActivity)
} else {
const videoObject = video.toActivityPubObject()
- const createActivity = createActivityData(video.url, byActor, videoObject, createActivityAudience)
+ const createActivity = buildCreateActivity(video.url, byActor, videoObject, createActivityAudience)
activities.push(createActivity)
}
import { VideoChannelsSearchQuery, VideosSearchQuery } from '../../../shared/models/search'
import { getOrCreateActorAndServerAndModel, getOrCreateVideoAndAccountAndChannel } from '../../lib/activitypub'
import { logger } from '../../helpers/logger'
-import { User } from '../../../shared/models/users'
-import { CONFIG } from '../../initializers/constants'
import { VideoChannelModel } from '../../models/video/video-channel'
import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger'
await sequelizeTypescript.transaction(async t => {
if (follow.state === 'accepted') await sendUndoFollow(follow, t)
+ // Disable redundancy on unfollowed instances
+ const server = follow.ActorFollowing.Server
+ server.redundancyAllowed = false
+ await server.save({ transaction: t })
+
await follow.destroy({ transaction: t })
})
import * as express from 'express'
import { serverFollowsRouter } from './follows'
import { statsRouter } from './stats'
+import { serverRedundancyRouter } from './redundancy'
const serverRouter = express.Router()
serverRouter.use('/', serverFollowsRouter)
+serverRouter.use('/', serverRedundancyRouter)
serverRouter.use('/', statsRouter)
// ---------------------------------------------------------------------------
--- /dev/null
+import * as express from 'express'
+import { UserRight } from '../../../../shared/models/users'
+import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares'
+import { updateServerRedundancyValidator } from '../../../middlewares/validators/redundancy'
+import { ServerModel } from '../../../models/server/server'
+
+const serverRedundancyRouter = express.Router()
+
+serverRedundancyRouter.put('/redundancy/:host',
+ authenticate,
+ ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW),
+ asyncMiddleware(updateServerRedundancyValidator),
+ asyncMiddleware(updateRedundancy)
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+ serverRedundancyRouter
+}
+
+// ---------------------------------------------------------------------------
+
+async function updateRedundancy (req: express.Request, res: express.Response, next: express.NextFunction) {
+ const server = res.locals.server as ServerModel
+
+ server.redundancyAllowed = req.body.redundancyAllowed
+
+ await server.save()
+
+ return res.sendStatus(204)
+}
// We send the video abuse to the origin server
if (videoInstance.isOwned() === false) {
- await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance, t)
+ await sendVideoAbuse(reporterAccount.Actor, videoAbuseInstance, videoInstance)
}
auditLogger.create(reporterAccount.Actor.getIdentifier(), new VideoAbuseAuditView(videoAbuseInstance.toFormattedJSON()))
'https://w3id.org/security/v1',
{
RsaSignature2017: 'https://w3id.org/security#RsaSignature2017',
+ pt: 'https://joinpeertube.org/ns',
+ schema: 'http://schema.org#',
Hashtag: 'as:Hashtag',
- uuid: 'http://schema.org/identifier',
- category: 'http://schema.org/category',
- licence: 'http://schema.org/license',
- subtitleLanguage: 'http://schema.org/subtitleLanguage',
+ uuid: 'schema:identifier',
+ category: 'schema:category',
+ licence: 'schema:license',
+ subtitleLanguage: 'schema:subtitleLanguage',
sensitive: 'as:sensitive',
- language: 'http://schema.org/inLanguage',
- views: 'http://schema.org/Number',
- stats: 'http://schema.org/Number',
- size: 'http://schema.org/Number',
- fps: 'http://schema.org/Number',
- commentsEnabled: 'http://schema.org/Boolean',
- waitTranscoding: 'http://schema.org/Boolean',
- support: 'http://schema.org/Text'
+ language: 'schema:inLanguage',
+ views: 'schema:Number',
+ stats: 'schema:Number',
+ size: 'schema:Number',
+ fps: 'schema:Number',
+ commentsEnabled: 'schema:Boolean',
+ waitTranscoding: 'schema:Boolean',
+ expires: 'schema:expires',
+ support: 'schema:Text',
+ CacheFile: 'pt:CacheFile'
},
{
likes: {
import * as validator from 'validator'
import { Activity, ActivityType } from '../../../../shared/models/activitypub'
import {
- isActorAcceptActivityValid, isActorDeleteActivityValid, isActorFollowActivityValid, isActorRejectActivityValid,
+ isActorAcceptActivityValid,
+ isActorDeleteActivityValid,
+ isActorFollowActivityValid,
+ isActorRejectActivityValid,
isActorUpdateActivityValid
} from './actor'
import { isAnnounceActivityValid } from './announce'
import { isVideoCommentCreateActivityValid, isVideoCommentDeleteActivityValid } from './video-comments'
import {
isVideoFlagValid,
- sanitizeAndCheckVideoTorrentCreateActivity,
isVideoTorrentDeleteActivityValid,
+ sanitizeAndCheckVideoTorrentCreateActivity,
sanitizeAndCheckVideoTorrentUpdateActivity
} from './videos'
import { isViewActivityValid } from './view'
import { exists } from '../misc'
+import { isCacheFileCreateActivityValid, isCacheFileUpdateActivityValid } from './cache-file'
function isRootActivityValid (activity: any) {
return Array.isArray(activity['@context']) && (
isDislikeActivityValid(activity) ||
sanitizeAndCheckVideoTorrentCreateActivity(activity) ||
isVideoFlagValid(activity) ||
- isVideoCommentCreateActivityValid(activity)
+ isVideoCommentCreateActivityValid(activity) ||
+ isCacheFileCreateActivityValid(activity)
}
function checkUpdateActivity (activity: any) {
- return sanitizeAndCheckVideoTorrentUpdateActivity(activity) ||
+ return isCacheFileUpdateActivityValid(activity) ||
+ sanitizeAndCheckVideoTorrentUpdateActivity(activity) ||
isActorUpdateActivityValid(activity)
}
--- /dev/null
+import { isActivityPubUrlValid, isBaseActivityValid } from './misc'
+import { isRemoteVideoUrlValid } from './videos'
+import { isDateValid, exists } from '../misc'
+import { CacheFileObject } from '../../../../shared/models/activitypub/objects'
+
+function isCacheFileCreateActivityValid (activity: any) {
+ return isBaseActivityValid(activity, 'Create') &&
+ isCacheFileObjectValid(activity.object)
+}
+
+function isCacheFileUpdateActivityValid (activity: any) {
+ return isBaseActivityValid(activity, 'Update') &&
+ isCacheFileObjectValid(activity.object)
+}
+
+function isCacheFileObjectValid (object: CacheFileObject) {
+ return exists(object) &&
+ object.type === 'CacheFile' &&
+ isDateValid(object.expires) &&
+ isActivityPubUrlValid(object.object) &&
+ isRemoteVideoUrlValid(object.url)
+}
+
+export {
+ isCacheFileUpdateActivityValid,
+ isCacheFileCreateActivityValid,
+ isCacheFileObjectValid
+}
import { isTestInstance } from '../../core-utils'
import { exists } from '../misc'
-function isActivityPubUrlValid (url: string) {
+function isUrlValid (url: string) {
const isURLOptions = {
require_host: true,
require_tld: true,
isURLOptions.require_tld = false
}
- return exists(url) && validator.isURL('' + url, isURLOptions) && validator.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL)
+ return exists(url) && validator.isURL('' + url, isURLOptions)
+}
+
+function isActivityPubUrlValid (url: string) {
+ return isUrlValid(url) && validator.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL)
}
function isBaseActivityValid (activity: any, type: string) {
return (activity['@context'] === undefined || Array.isArray(activity['@context'])) &&
activity.type === type &&
isActivityPubUrlValid(activity.id) &&
+ exists(activity.actor) &&
(isActivityPubUrlValid(activity.actor) || isActivityPubUrlValid(activity.actor.id)) &&
(
activity.to === undefined ||
}
export {
+ isUrlValid,
isActivityPubUrlValid,
isBaseActivityValid,
setValidAttributedTo
import { isBaseActivityValid } from './misc'
import { isDislikeActivityValid, isLikeActivityValid } from './rate'
import { isAnnounceActivityValid } from './announce'
+import { isCacheFileCreateActivityValid } from './cache-file'
function isUndoActivityValid (activity: any) {
return isBaseActivityValid(activity, 'Undo') &&
isActorFollowActivityValid(activity.object) ||
isLikeActivityValid(activity.object) ||
isDislikeActivityValid(activity.object) ||
- isAnnounceActivityValid(activity.object)
+ isAnnounceActivityValid(activity.object) ||
+ isCacheFileCreateActivityValid(activity.object)
)
}
video.attributedTo.length !== 0
}
+function isRemoteVideoUrlValid (url: any) {
+ // FIXME: Old bug, we used the width to represent the resolution. Remove it in a few release (currently beta.11)
+ if (url.width && !url.height) url.height = url.width
+
+ return url.type === 'Link' &&
+ (
+ ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mimeType) !== -1 &&
+ isActivityPubUrlValid(url.href) &&
+ validator.isInt(url.height + '', { min: 0 }) &&
+ validator.isInt(url.size + '', { min: 0 }) &&
+ (!url.fps || validator.isInt(url.fps + '', { min: 0 }))
+ ) ||
+ (
+ ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mimeType) !== -1 &&
+ isActivityPubUrlValid(url.href) &&
+ validator.isInt(url.height + '', { min: 0 })
+ ) ||
+ (
+ ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mimeType) !== -1 &&
+ validator.isLength(url.href, { min: 5 }) &&
+ validator.isInt(url.height + '', { min: 0 })
+ )
+}
+
// ---------------------------------------------------------------------------
export {
isVideoTorrentDeleteActivityValid,
isRemoteStringIdentifierValid,
isVideoFlagValid,
- sanitizeAndCheckVideoTorrentObject
+ sanitizeAndCheckVideoTorrentObject,
+ isRemoteVideoUrlValid
}
// ---------------------------------------------------------------------------
return true
}
-function isRemoteVideoUrlValid (url: any) {
- // FIXME: Old bug, we used the width to represent the resolution. Remove it in a few realease (currently beta.11)
- if (url.width && !url.height) url.height = url.width
- return url.type === 'Link' &&
- (
- ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.indexOf(url.mimeType) !== -1 &&
- isActivityPubUrlValid(url.href) &&
- validator.isInt(url.height + '', { min: 0 }) &&
- validator.isInt(url.size + '', { min: 0 }) &&
- (!url.fps || validator.isInt(url.fps + '', { min: 0 }))
- ) ||
- (
- ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.indexOf(url.mimeType) !== -1 &&
- isActivityPubUrlValid(url.href) &&
- validator.isInt(url.height + '', { min: 0 })
- ) ||
- (
- ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.indexOf(url.mimeType) !== -1 &&
- validator.isLength(url.href, { min: 5 }) &&
- validator.isInt(url.height + '', { min: 0 })
- )
-}
import { CONFIG } from '../initializers'
import { join } from 'path'
-function downloadWebTorrentVideo (target: { magnetUri: string, torrentName: string }) {
+function downloadWebTorrentVideo (target: { magnetUri: string, torrentName?: string }, timeout?: number) {
const id = target.magnetUri || target.torrentName
+ let timer
const path = generateVideoTmpPath(id)
logger.info('Importing torrent video %s', id)
return new Promise<string>((res, rej) => {
const webtorrent = new WebTorrent()
+ let file: WebTorrent.TorrentFile
const torrentId = target.magnetUri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName)
const options = { path: CONFIG.STORAGE.VIDEOS_DIR }
const torrent = webtorrent.add(torrentId, options, torrent => {
- if (torrent.files.length !== 1) return rej(new Error('The number of files is not equal to 1 for ' + torrentId))
+ if (torrent.files.length !== 1) {
+ if (timer) clearTimeout(timer)
- const file = torrent.files[ 0 ]
+ return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName)
+ .then(() => rej(new Error('The number of files is not equal to 1 for ' + torrentId)))
+ }
+
+ file = torrent.files[ 0 ]
const writeStream = createWriteStream(path)
writeStream.on('finish', () => {
- webtorrent.destroy(async err => {
- if (err) return rej(err)
-
- if (target.torrentName) {
- remove(torrentId)
- .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err }))
- }
+ if (timer) clearTimeout(timer)
- remove(join(CONFIG.STORAGE.VIDEOS_DIR, file.name))
- .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', file.name, { err }))
-
- res(path)
- })
+ return safeWebtorrentDestroy(webtorrent, torrentId, file.name, target.torrentName)
+ .then(() => res(path))
})
file.createReadStream().pipe(writeStream)
})
torrent.on('error', err => rej(err))
+
+ if (timeout) {
+ timer = setTimeout(async () => {
+ return safeWebtorrentDestroy(webtorrent, torrentId, file ? file.name : undefined, target.torrentName)
+ .then(() => rej(new Error('Webtorrent download timeout.')))
+ }, timeout)
+ }
})
}
export {
downloadWebTorrentVideo
}
+
+// ---------------------------------------------------------------------------
+
+function safeWebtorrentDestroy (webtorrent: WebTorrent.Instance, torrentId: string, filename?: string, torrentName?: string) {
+ return new Promise(res => {
+ webtorrent.destroy(err => {
+ // Delete torrent file
+ if (torrentName) {
+ remove(torrentId)
+ .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err }))
+ }
+
+ // Delete downloaded file
+ if (filename) {
+ remove(join(CONFIG.STORAGE.VIDEOS_DIR, filename))
+ .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', filename, { err }))
+ }
+
+ if (err) {
+ logger.warn('Cannot destroy webtorrent in timeout.', { err })
+ }
+
+ return res()
+ })
+ })
+}
import { CONFIG } from './constants'
import { logger } from '../helpers/logger'
import { getServerActor } from '../helpers/utils'
+import { VideosRedundancy } from '../../shared/models/redundancy'
+import { isArray } from '../helpers/custom-validators/misc'
+import { uniq } from 'lodash'
async function checkActivityPubUrls () {
const actor = await getServerActor()
return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy
}
+ const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos')
+ if (isArray(redundancyVideos)) {
+ for (const r of redundancyVideos) {
+ if ([ 'most-views' ].indexOf(r.strategy) === -1) {
+ return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy
+ }
+ }
+
+ const filtered = uniq(redundancyVideos.map(r => r.strategy))
+ if (filtered.length !== redundancyVideos.length) {
+ return 'Redundancy video entries should have uniq strategies'
+ }
+ }
+
return null
}
import { IConfig } from 'config'
import { dirname, join } from 'path'
-import { JobType, VideoRateType, VideoState } from '../../shared/models'
+import { JobType, VideoRateType, VideoRedundancyStrategy, VideoState, VideosRedundancy } from '../../shared/models'
import { ActivityPubActorType } from '../../shared/models/activitypub'
import { FollowState } from '../../shared/models/actors'
import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos'
import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
import { invert } from 'lodash'
import { CronRepeatOptions, EveryRepeatOptions } from 'bull'
+import * as bytes from 'bytes'
// Use a variable to reload the configuration if we need
let config: IConfig = require('config')
// ---------------------------------------------------------------------------
-const LAST_MIGRATION_VERSION = 265
+const LAST_MIGRATION_VERSION = 270
// ---------------------------------------------------------------------------
badActorFollow: 60000 * 60, // 1 hour
removeOldJobs: 60000 * 60, // 1 hour
updateVideos: 60000, // 1 minute
- youtubeDLUpdate: 60000 * 60 * 24 // 1 day
+ youtubeDLUpdate: 60000 * 60 * 24, // 1 day
+ videosRedundancy: 60000 * 2 // 2 hours
}
// ---------------------------------------------------------------------------
INTERVAL_DAYS: config.get<number>('trending.videos.interval_days')
}
},
+ REDUNDANCY: {
+ VIDEOS: buildVideosRedundancy(config.get<any[]>('redundancy.videos'))
+ },
ADMIN: {
get EMAIL () { return config.get<string>('admin.email') }
},
}
}
},
+ VIDEOS_REDUNDANCY: {
+ URL: { min: 3, max: 2000 } // Length
+ },
VIDEOS: {
NAME: { min: 3, max: 120 }, // Length
LANGUAGE: { min: 1, max: 10 }, // Length
}
}
+const REDUNDANCY = {
+ VIDEOS: {
+ EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days
+ RANDOMIZED_FACTOR: 5
+ }
+}
+
const ACCEPT_HEADERS = [ 'html', 'application/json' ].concat(ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS)
// ---------------------------------------------------------------------------
SCHEDULER_INTERVALS_MS.badActorFollow = 10000
SCHEDULER_INTERVALS_MS.removeOldJobs = 10000
SCHEDULER_INTERVALS_MS.updateVideos = 5000
+ SCHEDULER_INTERVALS_MS.videosRedundancy = 5000
REPEAT_JOBS['videos-views'] = { every: 5000 }
+ REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1
+
VIDEO_VIEW_LIFETIME = 1000 // 1 second
JOB_ATTEMPTS['email'] = 1
CONFIG,
CONSTRAINTS_FIELDS,
EMBED_SIZE,
+ REDUNDANCY,
JOB_CONCURRENCY,
JOB_ATTEMPTS,
LAST_MIGRATION_VERSION,
CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP)
}
+function buildVideosRedundancy (objs: { strategy: VideoRedundancyStrategy, size: string }[]): VideosRedundancy[] {
+ if (!objs) return []
+
+ return objs.map(obj => {
+ return {
+ strategy: obj.strategy,
+ size: bytes.parse(obj.size)
+ }
+ })
+}
+
function buildLanguages () {
const iso639 = require('iso-639-3')
import { VideoImportModel } from '../models/video/video-import'
import { VideoViewModel } from '../models/video/video-views'
import { VideoChangeOwnershipModel } from '../models/video/video-change-ownership'
+import { VideoRedundancyModel } from '../models/redundancy/video-redundancy'
require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
VideoCommentModel,
ScheduleVideoUpdateModel,
VideoImportModel,
- VideoViewModel
+ VideoViewModel,
+ VideoRedundancyModel
])
// Check extensions exist in the database
--- /dev/null
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+ transaction: Sequelize.Transaction
+ queryInterface: Sequelize.QueryInterface
+ sequelize: Sequelize.Sequelize
+}): Promise<any> {
+ {
+ const data = {
+ type: Sequelize.BOOLEAN,
+ allowNull: false,
+ defaultValue: false
+ }
+
+ await utils.queryInterface.addColumn('server', 'redundancyAllowed', data)
+ }
+
+}
+
+function down (options) {
+ throw new Error('Not implemented.')
+}
+
+export { up, down }
await actor.save({ transaction: t })
if (actor.Account) {
- await actor.save({ transaction: t })
-
actor.Account.set('name', result.name)
actor.Account.set('description', result.summary)
+
await actor.Account.save({ transaction: t })
} else if (actor.VideoChannel) {
- await actor.save({ transaction: t })
-
actor.VideoChannel.set('name', result.name)
actor.VideoChannel.set('description', result.summary)
actor.VideoChannel.set('support', result.support)
+
await actor.VideoChannel.save({ transaction: t })
}
--- /dev/null
+import { CacheFileObject } from '../../../shared/index'
+import { VideoModel } from '../../models/video/video'
+import { ActorModel } from '../../models/activitypub/actor'
+import { sequelizeTypescript } from '../../initializers'
+import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
+
+function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) {
+ const url = cacheFileObject.url
+
+ const videoFile = video.VideoFiles.find(f => {
+ return f.resolution === url.height && f.fps === url.fps
+ })
+
+ if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`)
+
+ return {
+ expiresOn: new Date(cacheFileObject.expires),
+ url: cacheFileObject.id,
+ fileUrl: cacheFileObject.url.href,
+ strategy: null,
+ videoFileId: videoFile.id,
+ actorId: byActor.id
+ }
+}
+
+function createCacheFile (cacheFileObject: CacheFileObject, video: VideoModel, byActor: ActorModel) {
+ return sequelizeTypescript.transaction(async t => {
+ const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor)
+
+ return VideoRedundancyModel.create(attributes, { transaction: t })
+ })
+}
+
+function updateCacheFile (cacheFileObject: CacheFileObject, redundancyModel: VideoRedundancyModel, byActor: ActorModel) {
+ const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, redundancyModel.VideoFile.Video, byActor)
+
+ redundancyModel.set('expires', attributes.expiresOn)
+ redundancyModel.set('fileUrl', attributes.fileUrl)
+
+ return redundancyModel.save()
+}
+
+export {
+ createCacheFile,
+ updateCacheFile,
+ cacheFileActivityObjectToDBAttributes
+}
-import { ActivityCreate, VideoAbuseState, VideoTorrentObject } from '../../../../shared'
+import { ActivityCreate, CacheFileObject, VideoAbuseState, VideoTorrentObject } from '../../../../shared'
import { DislikeObject, VideoAbuseObject, ViewObject } from '../../../../shared/models/activitypub/objects'
import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object'
import { retryTransactionWrapper } from '../../../helpers/database-utils'
import { getOrCreateVideoAndAccountAndChannel } from '../videos'
import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils'
import { Redis } from '../../redis'
+import { createCacheFile } from '../cache-file'
async function processCreateActivity (activity: ActivityCreate) {
const activityObject = activity.object
return retryTransactionWrapper(processCreateVideoAbuse, actor, activityObject as VideoAbuseObject)
} else if (activityType === 'Note') {
return retryTransactionWrapper(processCreateVideoComment, actor, activity)
+ } else if (activityType === 'CacheFile') {
+ return retryTransactionWrapper(processCacheFile, actor, activity)
}
logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id })
}
}
+async function processCacheFile (byActor: ActorModel, activity: ActivityCreate) {
+ const cacheFile = activity.object as CacheFileObject
+
+ const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFile.object)
+
+ await createCacheFile(cacheFile, video, byActor)
+
+ if (video.isOwned()) {
+ // Don't resend the activity to the sender
+ const exceptions = [ byActor ]
+ await forwardActivity(activity, undefined, exceptions)
+ }
+}
+
async function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) {
logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object)
state: VideoAbuseState.PENDING
}
- await VideoAbuseModel.create(videoAbuseData)
+ await VideoAbuseModel.create(videoAbuseData, { transaction: t })
logger.info('Remote abuse for video uuid %s created', videoAbuseToCreateData.object)
})
-import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo } from '../../../../shared/models/activitypub'
+import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo, CacheFileObject } from '../../../../shared/models/activitypub'
import { DislikeObject } from '../../../../shared/models/activitypub/objects'
import { getActorUrl } from '../../../helpers/activitypub'
import { retryTransactionWrapper } from '../../../helpers/database-utils'
import { forwardVideoRelatedActivity } from '../send/utils'
import { getOrCreateVideoAndAccountAndChannel } from '../videos'
import { VideoShareModel } from '../../../models/video/video-share'
+import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
async function processUndoActivity (activity: ActivityUndo) {
const activityToUndo = activity.object
if (activityToUndo.type === 'Like') {
return retryTransactionWrapper(processUndoLike, actorUrl, activity)
- } else if (activityToUndo.type === 'Create' && activityToUndo.object.type === 'Dislike') {
- return retryTransactionWrapper(processUndoDislike, actorUrl, activity)
- } else if (activityToUndo.type === 'Follow') {
+ }
+
+ if (activityToUndo.type === 'Create') {
+ if (activityToUndo.object.type === 'Dislike') {
+ return retryTransactionWrapper(processUndoDislike, actorUrl, activity)
+ } else if (activityToUndo.object.type === 'CacheFile') {
+ return retryTransactionWrapper(processUndoCacheFile, actorUrl, activity)
+ }
+ }
+
+ if (activityToUndo.type === 'Follow') {
return retryTransactionWrapper(processUndoFollow, actorUrl, activityToUndo)
- } else if (activityToUndo.type === 'Announce') {
+ }
+
+ if (activityToUndo.type === 'Announce') {
return retryTransactionWrapper(processUndoAnnounce, actorUrl, activityToUndo)
}
})
}
+async function processUndoCacheFile (actorUrl: string, activity: ActivityUndo) {
+ const cacheFileObject = activity.object.object as CacheFileObject
+
+ const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.object)
+
+ return sequelizeTypescript.transaction(async t => {
+ const byActor = await ActorModel.loadByUrl(actorUrl)
+ if (!byActor) throw new Error('Unknown actor ' + actorUrl)
+
+ const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id)
+ if (!cacheFile) throw new Error('Unknown video cache ' + cacheFile.url)
+
+ await cacheFile.destroy()
+
+ if (video.isOwned()) {
+ // Don't resend the activity to the sender
+ const exceptions = [ byActor ]
+
+ await forwardVideoRelatedActivity(activity, t, exceptions, video)
+ }
+ })
+}
+
function processUndoFollow (actorUrl: string, followActivity: ActivityFollow) {
return sequelizeTypescript.transaction(async t => {
const follower = await ActorModel.loadByUrl(actorUrl, t)
-import { ActivityUpdate, VideoTorrentObject } from '../../../../shared/models/activitypub'
+import { ActivityUpdate, CacheFileObject, VideoTorrentObject } from '../../../../shared/models/activitypub'
import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor'
import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils'
import { logger } from '../../../helpers/logger'
import { ActorModel } from '../../../models/activitypub/actor'
import { VideoChannelModel } from '../../../models/video/video-channel'
import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor'
-import { getOrCreateVideoAndAccountAndChannel, getOrCreateVideoChannelFromVideoObject, updateVideoFromAP } from '../videos'
+import { getOrCreateVideoAndAccountAndChannel, updateVideoFromAP, getOrCreateVideoChannelFromVideoObject } from '../videos'
import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos'
+import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file'
+import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
+import { createCacheFile, updateCacheFile } from '../cache-file'
async function processUpdateActivity (activity: ActivityUpdate) {
const actor = await getOrCreateActorAndServerAndModel(activity.actor)
if (objectType === 'Video') {
return retryTransactionWrapper(processUpdateVideo, actor, activity)
- } else if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') {
+ }
+
+ if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') {
return retryTransactionWrapper(processUpdateActor, actor, activity)
}
+ if (objectType === 'CacheFile') {
+ return retryTransactionWrapper(processUpdateCacheFile, actor, activity)
+ }
+
return undefined
}
const { video } = await getOrCreateVideoAndAccountAndChannel(videoObject.id)
const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
- return updateVideoFromAP(video, videoObject, actor, channelActor, activity.to)
+ return updateVideoFromAP(video, videoObject, actor.Account, channelActor.VideoChannel, activity.to)
+}
+
+async function processUpdateCacheFile (byActor: ActorModel, activity: ActivityUpdate) {
+ const cacheFileObject = activity.object as CacheFileObject
+
+ if (!isCacheFileObjectValid(cacheFileObject) === false) {
+ logger.debug('Cahe file object sent by update is not valid.', { cacheFileObject })
+ return undefined
+ }
+
+ const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id)
+ if (!redundancyModel) {
+ const { video } = await getOrCreateVideoAndAccountAndChannel(cacheFileObject.id)
+ return createCacheFile(cacheFileObject, video, byActor)
+ }
+
+ return updateCacheFile(cacheFileObject, redundancyModel, byActor)
}
async function processUpdateActor (actor: ActorModel, activity: ActivityUpdate) {
import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
import { getActorFollowAcceptActivityPubUrl, getActorFollowActivityPubUrl } from '../url'
import { unicastTo } from './utils'
-import { followActivityData } from './send-follow'
+import { buildFollowActivity } from './send-follow'
import { logger } from '../../../helpers/logger'
async function sendAccept (actorFollow: ActorFollowModel) {
logger.info('Creating job to accept follower %s.', follower.url)
const followUrl = getActorFollowActivityPubUrl(actorFollow)
- const followData = followActivityData(followUrl, follower, me)
+ const followData = buildFollowActivity(followUrl, follower, me)
const url = getActorFollowAcceptActivityPubUrl(actorFollow)
- const data = acceptActivityData(url, me, followData)
+ const data = buildAcceptActivity(url, me, followData)
return unicastTo(data, me, follower.inboxUrl)
}
// ---------------------------------------------------------------------------
-function acceptActivityData (url: string, byActor: ActorModel, followActivityData: ActivityFollow): ActivityAccept {
+function buildAcceptActivity (url: string, byActor: ActorModel, followActivityData: ActivityFollow): ActivityAccept {
return {
type: 'Accept',
id: url,
import { VideoModel } from '../../../models/video/video'
import { VideoShareModel } from '../../../models/video/video-share'
import { broadcastToFollowers } from './utils'
-import { getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience'
+import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience'
import { logger } from '../../../helpers/logger'
-async function buildVideoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
+async function buildAnnounceWithVideoAudience (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
const announcedObject = video.url
- const accountsToForwardView = await getActorsInvolvedInVideo(video, t)
- const audience = getObjectFollowersAudience(accountsToForwardView)
- return announceActivityData(videoShare.url, byActor, announcedObject, audience)
+ const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
+ const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
+
+ const activity = buildAnnounceActivity(videoShare.url, byActor, announcedObject, audience)
+
+ return { activity, actorsInvolvedInVideo }
}
async function sendVideoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
- const data = await buildVideoAnnounce(byActor, videoShare, video, t)
+ const { activity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, t)
logger.info('Creating job to send announce %s.', videoShare.url)
- const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
const followersException = [ byActor ]
-
- return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException)
+ return broadcastToFollowers(activity, byActor, actorsInvolvedInVideo, t, followersException)
}
-function announceActivityData (url: string, byActor: ActorModel, object: string, audience?: ActivityAudience): ActivityAnnounce {
+function buildAnnounceActivity (url: string, byActor: ActorModel, object: string, audience?: ActivityAudience): ActivityAnnounce {
if (!audience) audience = getAudience(byActor)
- return {
- type: 'Announce',
- to: audience.to,
- cc: audience.cc,
+ return audiencify({
+ type: 'Announce' as 'Announce',
id: url,
actor: byActor.url,
object
- }
+ }, audience)
}
// ---------------------------------------------------------------------------
export {
sendVideoAnnounce,
- announceActivityData,
- buildVideoAnnounce
+ buildAnnounceActivity,
+ buildAnnounceWithVideoAudience
}
getVideoCommentAudience
} from '../audience'
import { logger } from '../../../helpers/logger'
+import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
async function sendCreateVideo (video: VideoModel, t: Transaction) {
if (video.privacy === VideoPrivacy.PRIVATE) return undefined
const videoObject = video.toActivityPubObject()
const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC)
- const data = createActivityData(video.url, byActor, videoObject, audience)
+ const createActivity = buildCreateActivity(video.url, byActor, videoObject, audience)
- return broadcastToFollowers(data, byActor, [ byActor ], t)
+ return broadcastToFollowers(createActivity, byActor, [ byActor ], t)
}
-async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel, t: Transaction) {
+async function sendVideoAbuse (byActor: ActorModel, videoAbuse: VideoAbuseModel, video: VideoModel) {
if (!video.VideoChannel.Account.Actor.serverId) return // Local
const url = getVideoAbuseActivityPubUrl(videoAbuse)
logger.info('Creating job to send video abuse %s.', url)
const audience = { to: [ video.VideoChannel.Account.Actor.url ], cc: [] }
- const data = createActivityData(url, byActor, videoAbuse.toActivityPubObject(), audience)
+ const createActivity = buildCreateActivity(url, byActor, videoAbuse.toActivityPubObject(), audience)
- return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
+ return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
+}
+
+async function sendCreateCacheFile (byActor: ActorModel, fileRedundancy: VideoRedundancyModel) {
+ logger.info('Creating job to send file cache of %s.', fileRedundancy.url)
+
+ const redundancyObject = fileRedundancy.toActivityPubObject()
+
+ const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(fileRedundancy.VideoFile.Video.id)
+ const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined)
+
+ const audience = getVideoAudience(video, actorsInvolvedInVideo)
+ const createActivity = buildCreateActivity(fileRedundancy.url, byActor, redundancyObject, audience)
+
+ return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
}
async function sendCreateVideoComment (comment: VideoCommentModel, t: Transaction) {
audience = getObjectFollowersAudience(actorsInvolvedInComment.concat(parentsCommentActors))
}
- const data = createActivityData(comment.url, byActor, commentObject, audience)
+ const createActivity = buildCreateActivity(comment.url, byActor, commentObject, audience)
// This was a reply, send it to the parent actors
const actorsException = [ byActor ]
- await broadcastToActors(data, byActor, parentsCommentActors, actorsException)
+ await broadcastToActors(createActivity, byActor, parentsCommentActors, actorsException)
// Broadcast to our followers
- await broadcastToFollowers(data, byActor, [ byActor ], t)
+ await broadcastToFollowers(createActivity, byActor, [ byActor ], t)
// Send to actors involved in the comment
- if (isOrigin) return broadcastToFollowers(data, byActor, actorsInvolvedInComment, t, actorsException)
+ if (isOrigin) return broadcastToFollowers(createActivity, byActor, actorsInvolvedInComment, t, actorsException)
// Send to origin
- return unicastTo(data, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
+ return unicastTo(createActivity, byActor, comment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
}
async function sendCreateView (byActor: ActorModel, video: VideoModel, t: Transaction) {
logger.info('Creating job to send view of %s.', video.url)
const url = getVideoViewActivityPubUrl(byActor, video)
- const viewActivityData = createViewActivityData(byActor, video)
+ const viewActivity = buildViewActivity(byActor, video)
const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
// Send to origin
if (video.isOwned() === false) {
const audience = getVideoAudience(video, actorsInvolvedInVideo)
- const data = createActivityData(url, byActor, viewActivityData, audience)
+ const createActivity = buildCreateActivity(url, byActor, viewActivity, audience)
- return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
+ return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
}
// Send to followers
const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
- const data = createActivityData(url, byActor, viewActivityData, audience)
+ const createActivity = buildCreateActivity(url, byActor, viewActivity, audience)
// Use the server actor to send the view
const serverActor = await getServerActor()
const actorsException = [ byActor ]
- return broadcastToFollowers(data, serverActor, actorsInvolvedInVideo, t, actorsException)
+ return broadcastToFollowers(createActivity, serverActor, actorsInvolvedInVideo, t, actorsException)
}
async function sendCreateDislike (byActor: ActorModel, video: VideoModel, t: Transaction) {
logger.info('Creating job to dislike %s.', video.url)
const url = getVideoDislikeActivityPubUrl(byActor, video)
- const dislikeActivityData = createDislikeActivityData(byActor, video)
+ const dislikeActivity = buildDislikeActivity(byActor, video)
const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
// Send to origin
if (video.isOwned() === false) {
const audience = getVideoAudience(video, actorsInvolvedInVideo)
- const data = createActivityData(url, byActor, dislikeActivityData, audience)
+ const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience)
- return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
+ return unicastTo(createActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
}
// Send to followers
const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
- const data = createActivityData(url, byActor, dislikeActivityData, audience)
+ const createActivity = buildCreateActivity(url, byActor, dislikeActivity, audience)
const actorsException = [ byActor ]
- return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, actorsException)
+ return broadcastToFollowers(createActivity, byActor, actorsInvolvedInVideo, t, actorsException)
}
-function createActivityData (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate {
+function buildCreateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityCreate {
if (!audience) audience = getAudience(byActor)
return audiencify(
)
}
-function createDislikeActivityData (byActor: ActorModel, video: VideoModel) {
+function buildDislikeActivity (byActor: ActorModel, video: VideoModel) {
return {
type: 'Dislike',
actor: byActor.url,
}
}
-function createViewActivityData (byActor: ActorModel, video: VideoModel) {
+function buildViewActivity (byActor: ActorModel, video: VideoModel) {
return {
type: 'View',
actor: byActor.url,
export {
sendCreateVideo,
sendVideoAbuse,
- createActivityData,
+ buildCreateActivity,
sendCreateView,
sendCreateDislike,
- createDislikeActivityData,
- sendCreateVideoComment
+ buildDislikeActivity,
+ sendCreateVideoComment,
+ sendCreateCacheFile
}
const url = getDeleteActivityPubUrl(video.url)
const byActor = video.VideoChannel.Account.Actor
- const data = deleteActivityData(url, video.url, byActor)
+ const activity = buildDeleteActivity(url, video.url, byActor)
- const actorsInvolved = await VideoShareModel.loadActorsByShare(video.id, t)
- actorsInvolved.push(byActor)
+ const actorsInvolved = await getActorsInvolvedInVideo(video, t)
- return broadcastToFollowers(data, byActor, actorsInvolved, t)
+ return broadcastToFollowers(activity, byActor, actorsInvolved, t)
}
async function sendDeleteActor (byActor: ActorModel, t: Transaction) {
logger.info('Creating job to broadcast delete of actor %s.', byActor.url)
const url = getDeleteActivityPubUrl(byActor.url)
- const data = deleteActivityData(url, byActor.url, byActor)
+ const activity = buildDeleteActivity(url, byActor.url, byActor)
const actorsInvolved = await VideoShareModel.loadActorsByVideoOwner(byActor.id, t)
actorsInvolved.push(byActor)
- return broadcastToFollowers(data, byActor, actorsInvolved, t)
+ return broadcastToFollowers(activity, byActor, actorsInvolved, t)
}
async function sendDeleteVideoComment (videoComment: VideoCommentModel, t: Transaction) {
const threadParentComments = await VideoCommentModel.listThreadParentComments(videoComment, t)
const actorsInvolvedInComment = await getActorsInvolvedInVideo(videoComment.Video, t)
- actorsInvolvedInComment.push(byActor)
+ actorsInvolvedInComment.push(byActor) // Add the actor that commented the video
const audience = getVideoCommentAudience(videoComment, threadParentComments, actorsInvolvedInComment, isVideoOrigin)
- const data = deleteActivityData(url, videoComment.url, byActor, audience)
+ const activity = buildDeleteActivity(url, videoComment.url, byActor, audience)
// This was a reply, send it to the parent actors
const actorsException = [ byActor ]
- await broadcastToActors(data, byActor, threadParentComments.map(c => c.Account.Actor), actorsException)
+ await broadcastToActors(activity, byActor, threadParentComments.map(c => c.Account.Actor), actorsException)
// Broadcast to our followers
- await broadcastToFollowers(data, byActor, [ byActor ], t)
+ await broadcastToFollowers(activity, byActor, [ byActor ], t)
// Send to actors involved in the comment
- if (isVideoOrigin) return broadcastToFollowers(data, byActor, actorsInvolvedInComment, t, actorsException)
+ if (isVideoOrigin) return broadcastToFollowers(activity, byActor, actorsInvolvedInComment, t, actorsException)
// Send to origin
- return unicastTo(data, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
+ return unicastTo(activity, byActor, videoComment.Video.VideoChannel.Account.Actor.sharedInboxUrl)
}
// ---------------------------------------------------------------------------
// ---------------------------------------------------------------------------
-function deleteActivityData (url: string, object: string, byActor: ActorModel, audience?: ActivityAudience): ActivityDelete {
+function buildDeleteActivity (url: string, object: string, byActor: ActorModel, audience?: ActivityAudience): ActivityDelete {
const activity = {
type: 'Delete' as 'Delete',
id: url,
logger.info('Creating job to send follow request to %s.', following.url)
const url = getActorFollowActivityPubUrl(actorFollow)
- const data = followActivityData(url, me, following)
+ const data = buildFollowActivity(url, me, following)
return unicastTo(data, me, following.inboxUrl)
}
-function followActivityData (url: string, byActor: ActorModel, targetActor: ActorModel): ActivityFollow {
+function buildFollowActivity (url: string, byActor: ActorModel, targetActor: ActorModel): ActivityFollow {
return {
type: 'Follow',
id: url,
export {
sendFollow,
- followActivityData
+ buildFollowActivity
}
// Send to origin
if (video.isOwned() === false) {
const audience = getVideoAudience(video, accountsInvolvedInVideo)
- const data = likeActivityData(url, byActor, video, audience)
+ const data = buildLikeActivity(url, byActor, video, audience)
return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
}
// Send to followers
const audience = getObjectFollowersAudience(accountsInvolvedInVideo)
- const data = likeActivityData(url, byActor, video, audience)
+ const activity = buildLikeActivity(url, byActor, video, audience)
const followersException = [ byActor ]
- return broadcastToFollowers(data, byActor, accountsInvolvedInVideo, t, followersException)
+ return broadcastToFollowers(activity, byActor, accountsInvolvedInVideo, t, followersException)
}
-function likeActivityData (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityLike {
+function buildLikeActivity (url: string, byActor: ActorModel, video: VideoModel, audience?: ActivityAudience): ActivityLike {
if (!audience) audience = getAudience(byActor)
return audiencify(
export {
sendLike,
- likeActivityData
+ buildLikeActivity
}
import { getActorFollowActivityPubUrl, getUndoActivityPubUrl, getVideoDislikeActivityPubUrl, getVideoLikeActivityPubUrl } from '../url'
import { broadcastToFollowers, unicastTo } from './utils'
import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience, getVideoAudience } from '../audience'
-import { createActivityData, createDislikeActivityData } from './send-create'
-import { followActivityData } from './send-follow'
-import { likeActivityData } from './send-like'
+import { buildCreateActivity, buildDislikeActivity } from './send-create'
+import { buildFollowActivity } from './send-follow'
+import { buildLikeActivity } from './send-like'
import { VideoShareModel } from '../../../models/video/video-share'
-import { buildVideoAnnounce } from './send-announce'
+import { buildAnnounceWithVideoAudience } from './send-announce'
import { logger } from '../../../helpers/logger'
+import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
async function sendUndoFollow (actorFollow: ActorFollowModel, t: Transaction) {
const me = actorFollow.ActorFollower
const followUrl = getActorFollowActivityPubUrl(actorFollow)
const undoUrl = getUndoActivityPubUrl(followUrl)
- const object = followActivityData(followUrl, me, following)
- const data = undoActivityData(undoUrl, me, object)
+ const followActivity = buildFollowActivity(followUrl, me, following)
+ const undoActivity = undoActivityData(undoUrl, me, followActivity)
- return unicastTo(data, me, following.inboxUrl)
+ return unicastTo(undoActivity, me, following.inboxUrl)
}
async function sendUndoLike (byActor: ActorModel, video: VideoModel, t: Transaction) {
const undoUrl = getUndoActivityPubUrl(likeUrl)
const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
- const object = likeActivityData(likeUrl, byActor, video)
+ const likeActivity = buildLikeActivity(likeUrl, byActor, video)
// Send to origin
if (video.isOwned() === false) {
const audience = getVideoAudience(video, actorsInvolvedInVideo)
- const data = undoActivityData(undoUrl, byActor, object, audience)
+ const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience)
- return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
+ return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
}
const audience = getObjectFollowersAudience(actorsInvolvedInVideo)
- const data = undoActivityData(undoUrl, byActor, object, audience)
+ const undoActivity = undoActivityData(undoUrl, byActor, likeActivity, audience)
const followersException = [ byActor ]
- return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException)
+ return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
}
async function sendUndoDislike (byActor: ActorModel, video: VideoModel, t: Transaction) {
const undoUrl = getUndoActivityPubUrl(dislikeUrl)
const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
- const dislikeActivity = createDislikeActivityData(byActor, video)
- const object = createActivityData(dislikeUrl, byActor, dislikeActivity)
+ const dislikeActivity = buildDislikeActivity(byActor, video)
+ const createDislikeActivity = buildCreateActivity(dislikeUrl, byActor, dislikeActivity)
if (video.isOwned() === false) {
const audience = getVideoAudience(video, actorsInvolvedInVideo)
- const data = undoActivityData(undoUrl, byActor, object, audience)
+ const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity, audience)
- return unicastTo(data, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
+ return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
}
- const data = undoActivityData(undoUrl, byActor, object)
+ const undoActivity = undoActivityData(undoUrl, byActor, createDislikeActivity)
const followersException = [ byActor ]
- return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException)
+ return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
}
async function sendUndoAnnounce (byActor: ActorModel, videoShare: VideoShareModel, video: VideoModel, t: Transaction) {
const undoUrl = getUndoActivityPubUrl(videoShare.url)
- const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
- const object = await buildVideoAnnounce(byActor, videoShare, video, t)
- const data = undoActivityData(undoUrl, byActor, object)
+ const { activity: announceActivity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, t)
+ const undoActivity = undoActivityData(undoUrl, byActor, announceActivity)
const followersException = [ byActor ]
- return broadcastToFollowers(data, byActor, actorsInvolvedInVideo, t, followersException)
+ return broadcastToFollowers(undoActivity, byActor, actorsInvolvedInVideo, t, followersException)
+}
+
+async function sendUndoCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel, t: Transaction) {
+ logger.info('Creating job to undo cache file %s.', redundancyModel.url)
+
+ const undoUrl = getUndoActivityPubUrl(redundancyModel.url)
+
+ const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id)
+ const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t)
+
+ const audience = getVideoAudience(video, actorsInvolvedInVideo)
+ const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject())
+
+ const undoActivity = undoActivityData(undoUrl, byActor, createActivity, audience)
+
+ return unicastTo(undoActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
}
// ---------------------------------------------------------------------------
sendUndoFollow,
sendUndoLike,
sendUndoDislike,
- sendUndoAnnounce
+ sendUndoAnnounce,
+ sendUndoCacheFile
}
// ---------------------------------------------------------------------------
import { VideoChannelModel } from '../../../models/video/video-channel'
import { VideoShareModel } from '../../../models/video/video-share'
import { getUpdateActivityPubUrl } from '../url'
-import { broadcastToFollowers } from './utils'
-import { audiencify, getAudience } from '../audience'
+import { broadcastToFollowers, unicastTo } from './utils'
+import { audiencify, getActorsInvolvedInVideo, getAudience, getObjectFollowersAudience } from '../audience'
import { logger } from '../../../helpers/logger'
-import { videoFeedsValidator } from '../../../middlewares/validators'
import { VideoCaptionModel } from '../../../models/video/video-caption'
+import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
async function sendUpdateVideo (video: VideoModel, t: Transaction, overrodeByActor?: ActorModel) {
logger.info('Creating job to update video %s.', video.url)
const videoObject = video.toActivityPubObject()
const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC)
- const data = updateActivityData(url, byActor, videoObject, audience)
+ const updateActivity = buildUpdateActivity(url, byActor, videoObject, audience)
- const actorsInvolved = await VideoShareModel.loadActorsByShare(video.id, t)
- actorsInvolved.push(byActor)
+ const actorsInvolved = await getActorsInvolvedInVideo(video, t)
+ if (overrodeByActor) actorsInvolved.push(overrodeByActor)
- return broadcastToFollowers(data, byActor, actorsInvolved, t)
+ return broadcastToFollowers(updateActivity, byActor, actorsInvolved, t)
}
async function sendUpdateActor (accountOrChannel: AccountModel | VideoChannelModel, t: Transaction) {
const url = getUpdateActivityPubUrl(byActor.url, byActor.updatedAt.toISOString())
const accountOrChannelObject = accountOrChannel.toActivityPubObject()
const audience = getAudience(byActor)
- const data = updateActivityData(url, byActor, accountOrChannelObject, audience)
+ const updateActivity = buildUpdateActivity(url, byActor, accountOrChannelObject, audience)
let actorsInvolved: ActorModel[]
if (accountOrChannel instanceof AccountModel) {
actorsInvolved.push(byActor)
- return broadcastToFollowers(data, byActor, actorsInvolved, t)
+ return broadcastToFollowers(updateActivity, byActor, actorsInvolved, t)
+}
+
+async function sendUpdateCacheFile (byActor: ActorModel, redundancyModel: VideoRedundancyModel) {
+ logger.info('Creating job to update cache file %s.', redundancyModel.url)
+
+ const url = getUpdateActivityPubUrl(redundancyModel.url, redundancyModel.updatedAt.toISOString())
+ const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(redundancyModel.VideoFile.Video.id)
+
+ const redundancyObject = redundancyModel.toActivityPubObject()
+
+ const accountsInvolvedInVideo = await getActorsInvolvedInVideo(video, undefined)
+ const audience = getObjectFollowersAudience(accountsInvolvedInVideo)
+
+ const updateActivity = buildUpdateActivity(url, byActor, redundancyObject, audience)
+ return unicastTo(updateActivity, byActor, video.VideoChannel.Account.Actor.sharedInboxUrl)
}
// ---------------------------------------------------------------------------
export {
sendUpdateActor,
- sendUpdateVideo
+ sendUpdateVideo,
+ sendUpdateCacheFile
}
// ---------------------------------------------------------------------------
-function updateActivityData (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityUpdate {
+function buildUpdateActivity (url: string, byActor: ActorModel, object: any, audience?: ActivityAudience): ActivityUpdate {
if (!audience) audience = getAudience(byActor)
return audiencify(
async function broadcastToFollowers (
data: any,
byActor: ActorModel,
- toActorFollowers: ActorModel[],
+ toFollowersOf: ActorModel[],
t: Transaction,
actorsException: ActorModel[] = []
) {
- const uris = await computeFollowerUris(toActorFollowers, actorsException, t)
+ const uris = await computeFollowerUris(toFollowersOf, actorsException, t)
return broadcastTo(uris, data, byActor)
}
// ---------------------------------------------------------------------------
-async function computeFollowerUris (toActorFollower: ActorModel[], actorsException: ActorModel[], t: Transaction) {
- const toActorFollowerIds = toActorFollower.map(a => a.id)
+async function computeFollowerUris (toFollowersOf: ActorModel[], actorsException: ActorModel[], t: Transaction) {
+ const toActorFollowerIds = toFollowersOf.map(a => a.id)
const result = await ActorFollowModel.listAcceptedFollowerSharedInboxUrls(toActorFollowerIds, t)
const sharedInboxesException = await buildSharedInboxesException(actorsException)
import { VideoModel } from '../../models/video/video'
import { VideoAbuseModel } from '../../models/video/video-abuse'
import { VideoCommentModel } from '../../models/video/video-comment'
+import { VideoFileModel } from '../../models/video/video-file'
function getVideoActivityPubUrl (video: VideoModel) {
return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid
}
+function getVideoCacheFileActivityPubUrl (videoFile: VideoFileModel) {
+ const suffixFPS = videoFile.fps ? '-' + videoFile.fps : ''
+
+ return `${CONFIG.WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}`
+}
+
function getVideoCommentActivityPubUrl (video: VideoModel, videoComment: VideoCommentModel) {
return CONFIG.WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id
}
getVideoSharesActivityPubUrl,
getVideoCommentsActivityPubUrl,
getVideoLikesActivityPubUrl,
- getVideoDislikesActivityPubUrl
+ getVideoDislikesActivityPubUrl,
+ getVideoCacheFileActivityPubUrl
}
import * as magnetUtil from 'magnet-uri'
import { join } from 'path'
import * as request from 'request'
-import { ActivityIconObject, VideoState } from '../../../shared/index'
+import { ActivityIconObject, ActivityVideoUrlObject, VideoState, ActivityUrlObject } from '../../../shared/index'
import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
import { VideoPrivacy } from '../../../shared/models/videos'
import { sanitizeAndCheckVideoTorrentObject } from '../../helpers/custom-validators/activitypub/videos'
import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
-import { resetSequelizeInstance, retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils'
+import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils'
import { logger } from '../../helpers/logger'
import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, VIDEO_MIMETYPE_EXT } from '../../initializers'
import { VideoModel } from '../../models/video/video'
import { VideoChannelModel } from '../../models/video/video-channel'
import { VideoFileModel } from '../../models/video/video-file'
-import { getOrCreateActorAndServerAndModel, updateActorAvatarInstance } from './actor'
+import { getOrCreateActorAndServerAndModel } from './actor'
import { addVideoComments } from './video-comments'
import { crawlCollectionPage } from './crawl'
import { sendCreateVideo, sendUpdateVideo } from './send'
import { VideoCaptionModel } from '../../models/video/video-caption'
import { JobQueue } from '../job-queue'
import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher'
-import { getUrlFromWebfinger } from '../../helpers/webfinger'
import { createRates } from './video-rates'
import { addVideoShares, shareVideoByServerAndChannel } from './share'
import { AccountModel } from '../../models/account/account'
}
function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) {
- const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT)
- const fileUrls = videoObject.url.filter(u => {
- return mimeTypes.indexOf(u.mimeType) !== -1 && u.mimeType.startsWith('video/')
- })
+ const fileUrls = videoObject.url.filter(u => isActivityVideoUrlObject(u)) as ActivityVideoUrlObject[]
if (fileUrls.length === 0) {
throw new Error('Cannot find video files for ' + videoCreated.url)
const channelActor = await getOrCreateVideoChannelFromVideoObject(videoObject)
const account = await AccountModel.load(channelActor.VideoChannel.accountId)
- return updateVideoFromAP(video, videoObject, account.Actor, channelActor)
+ return updateVideoFromAP(video, videoObject, account, channelActor.VideoChannel)
} catch (err) {
logger.warn('Cannot refresh video.', { err })
return video
async function updateVideoFromAP (
video: VideoModel,
videoObject: VideoTorrentObject,
- accountActor: ActorModel,
- channelActor: ActorModel,
+ account: AccountModel,
+ channel: VideoChannelModel,
overrideTo?: string[]
) {
logger.debug('Updating remote video "%s".', videoObject.uuid)
// Check actor has the right to update the video
const videoChannel = video.VideoChannel
- if (videoChannel.Account.Actor.id !== accountActor.id) {
- throw new Error('Account ' + accountActor.url + ' does not own video channel ' + videoChannel.Actor.url)
+ if (videoChannel.Account.id !== account.id) {
+ throw new Error('Account ' + account.Actor.url + ' does not own video channel ' + videoChannel.Actor.url)
}
const to = overrideTo ? overrideTo : videoObject.to
- const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, to)
+ const videoData = await videoActivityObjectToDBAttributes(channel, videoObject, to)
video.set('name', videoData.name)
video.set('uuid', videoData.uuid)
video.set('url', videoData.url)
addVideoShares,
createRates
}
+
+// ---------------------------------------------------------------------------
+
+function isActivityVideoUrlObject (url: ActivityUrlObject): url is ActivityVideoUrlObject {
+ const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT)
+
+ return mimeTypes.indexOf(url.mimeType) !== -1 && url.mimeType.startsWith('video/')
+}
--- /dev/null
+import { VideoRedundancyModel } from '../models/redundancy/video-redundancy'
+import { sendUndoCacheFile } from './activitypub/send'
+import { Transaction } from 'sequelize'
+import { getServerActor } from '../helpers/utils'
+
+async function removeVideoRedundancy (videoRedundancy: VideoRedundancyModel, t?: Transaction) {
+ const serverActor = await getServerActor()
+
+ await sendUndoCacheFile(serverActor, videoRedundancy, t)
+
+ await videoRedundancy.destroy({ transaction: t })
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+ removeVideoRedundancy
+}
--- /dev/null
+import { AbstractScheduler } from './abstract-scheduler'
+import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers'
+import { logger } from '../../helpers/logger'
+import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
+import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
+import { VideoFileModel } from '../../models/video/video-file'
+import { sortBy } from 'lodash'
+import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
+import { join } from 'path'
+import { rename } from 'fs-extra'
+import { getServerActor } from '../../helpers/utils'
+import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
+import { VideoModel } from '../../models/video/video'
+import { getVideoCacheFileActivityPubUrl } from '../activitypub/url'
+import { removeVideoRedundancy } from '../redundancy'
+import { isTestInstance } from '../../helpers/core-utils'
+
+export class VideosRedundancyScheduler extends AbstractScheduler {
+
+ private static instance: AbstractScheduler
+ private executing = false
+
+ protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.videosRedundancy
+
+ private constructor () {
+ super()
+ }
+
+ async execute () {
+ if (this.executing) return
+
+ this.executing = true
+
+ for (const obj of CONFIG.REDUNDANCY.VIDEOS) {
+
+ try {
+ const videoToDuplicate = await this.findVideoToDuplicate(obj.strategy)
+ if (!videoToDuplicate) continue
+
+ const videoFiles = videoToDuplicate.VideoFiles
+ videoFiles.forEach(f => f.Video = videoToDuplicate)
+
+ const videosRedundancy = await VideoRedundancyModel.getVideoFiles(obj.strategy)
+ if (this.isTooHeavy(videosRedundancy, videoFiles, obj.size)) {
+ if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url)
+ continue
+ }
+
+ logger.info('Will duplicate video %s in redundancy scheduler "%s".', videoToDuplicate.url, obj.strategy)
+
+ await this.createVideoRedundancy(obj.strategy, videoFiles)
+ } catch (err) {
+ logger.error('Cannot run videos redundancy %s.', obj.strategy, { err })
+ }
+ }
+
+ const expired = await VideoRedundancyModel.listAllExpired()
+
+ for (const m of expired) {
+ logger.info('Removing expired video %s from our redundancy system.', this.buildEntryLogId(m))
+
+ try {
+ await m.destroy()
+ } catch (err) {
+ logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m))
+ }
+ }
+
+ this.executing = false
+ }
+
+ static get Instance () {
+ return this.instance || (this.instance = new this())
+ }
+
+ private findVideoToDuplicate (strategy: VideoRedundancyStrategy) {
+ if (strategy === 'most-views') return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
+ }
+
+ private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) {
+ const serverActor = await getServerActor()
+
+ for (const file of filesToDuplicate) {
+ const existing = await VideoRedundancyModel.loadByFileId(file.id)
+ if (existing) {
+ logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', file.Video.url, file.resolution, strategy)
+
+ existing.expiresOn = this.buildNewExpiration()
+ await existing.save()
+
+ await sendUpdateCacheFile(serverActor, existing)
+ continue
+ }
+
+ // We need more attributes and check if the video still exists
+ const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(file.Video.id)
+ if (!video) continue
+
+ logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy)
+
+ const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
+ const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs)
+
+ const tmpPath = await downloadWebTorrentVideo({ magnetUri }, JOB_TTL['video-import'])
+
+ const destPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file))
+ await rename(tmpPath, destPath)
+
+ const createdModel = await VideoRedundancyModel.create({
+ expiresOn: new Date(Date.now() + REDUNDANCY.VIDEOS.EXPIRES_AFTER_MS),
+ url: getVideoCacheFileActivityPubUrl(file),
+ fileUrl: video.getVideoFileUrl(file, CONFIG.WEBSERVER.URL),
+ strategy,
+ videoFileId: file.id,
+ actorId: serverActor.id
+ })
+ createdModel.VideoFile = file
+
+ await sendCreateCacheFile(serverActor, createdModel)
+ }
+ }
+
+ // Unused, but could be useful in the future, with a custom strategy
+ private async purgeVideosIfNeeded (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSize: number) {
+ const sortedVideosRedundancy = sortBy(videosRedundancy, 'createdAt')
+
+ while (this.isTooHeavy(sortedVideosRedundancy, filesToDuplicate, maxSize)) {
+ const toDelete = sortedVideosRedundancy.shift()
+
+ const videoFile = toDelete.VideoFile
+ logger.info('Purging video %s (resolution %d) from our redundancy system.', videoFile.Video.url, videoFile.resolution)
+
+ await removeVideoRedundancy(toDelete, undefined)
+ }
+
+ return sortedVideosRedundancy
+ }
+
+ private isTooHeavy (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSizeArg: number) {
+ const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate)
+
+ const redundancyReducer = (previous: number, current: VideoRedundancyModel) => previous + current.VideoFile.size
+ const totalDuplicated = videosRedundancy.reduce(redundancyReducer, 0)
+
+ return totalDuplicated > maxSize
+ }
+
+ private buildNewExpiration () {
+ return new Date(Date.now() + REDUNDANCY.VIDEOS.EXPIRES_AFTER_MS)
+ }
+
+ private buildEntryLogId (object: VideoRedundancyModel) {
+ return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}`
+ }
+
+ private getTotalFileSizes (files: VideoFileModel[]) {
+ const fileReducer = (previous: number, current: VideoFileModel) => previous + current.size
+
+ return files.reduce(fileReducer, 0)
+ }
+}
--- /dev/null
+import * as express from 'express'
+import 'express-validator'
+import { param, body } from 'express-validator/check'
+import { exists, isBooleanValid, isIdOrUUIDValid, toIntOrNull } from '../../helpers/custom-validators/misc'
+import { isVideoExist } from '../../helpers/custom-validators/videos'
+import { logger } from '../../helpers/logger'
+import { areValidationErrors } from './utils'
+import { VideoModel } from '../../models/video/video'
+import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
+import { isHostValid } from '../../helpers/custom-validators/servers'
+import { getServerActor } from '../../helpers/utils'
+import { ActorFollowModel } from '../../models/activitypub/actor-follow'
+import { SERVER_ACTOR_NAME } from '../../initializers'
+import { ServerModel } from '../../models/server/server'
+
+const videoRedundancyGetValidator = [
+ param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'),
+ param('resolution')
+ .customSanitizer(toIntOrNull)
+ .custom(exists).withMessage('Should have a valid resolution'),
+ param('fps')
+ .optional()
+ .customSanitizer(toIntOrNull)
+ .custom(exists).withMessage('Should have a valid fps'),
+
+ async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+ logger.debug('Checking videoRedundancyGetValidator parameters', { parameters: req.params })
+
+ if (areValidationErrors(req, res)) return
+ if (!await isVideoExist(req.params.videoId, res)) return
+
+ const video: VideoModel = res.locals.video
+ const videoFile = video.VideoFiles.find(f => {
+ return f.resolution === req.params.resolution && (!req.params.fps || f.fps === req.params.fps)
+ })
+
+ if (!videoFile) return res.status(404).json({ error: 'Video file not found.' })
+ res.locals.videoFile = videoFile
+
+ const videoRedundancy = await VideoRedundancyModel.loadByFileId(videoFile.id)
+ if (!videoRedundancy)return res.status(404).json({ error: 'Video redundancy not found.' })
+ res.locals.videoRedundancy = videoRedundancy
+
+ return next()
+ }
+]
+
+const updateServerRedundancyValidator = [
+ param('host').custom(isHostValid).withMessage('Should have a valid host'),
+ body('redundancyAllowed')
+ .toBoolean()
+ .custom(isBooleanValid).withMessage('Should have a valid redundancyAllowed attribute'),
+
+ async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+ logger.debug('Checking updateServerRedundancy parameters', { parameters: req.params })
+
+ if (areValidationErrors(req, res)) return
+
+ const server = await ServerModel.loadByHost(req.params.host)
+
+ if (!server) {
+ return res
+ .status(404)
+ .json({
+ error: `Server ${req.params.host} not found.`
+ })
+ .end()
+ }
+
+ res.locals.server = server
+ return next()
+ }
+]
+
+// ---------------------------------------------------------------------------
+
+export {
+ videoRedundancyGetValidator,
+ updateServerRedundancyValidator
+}
UpdatedAt
} from 'sequelize-typescript'
import { FollowState } from '../../../shared/models/actors'
-import { AccountFollow } from '../../../shared/models/actors/follow.model'
+import { ActorFollow } from '../../../shared/models/actors/follow.model'
import { logger } from '../../helpers/logger'
import { getServerActor } from '../../helpers/utils'
import { ACTOR_FOLLOW_SCORE } from '../../initializers'
return ActorFollowModel.findAll(query)
}
- toFormattedJSON (): AccountFollow {
+ toFormattedJSON (): ActorFollow {
const follower = this.ActorFollower.toFormattedJSON()
const following = this.ActorFollowing.toFormattedJSON()
},
{
model: () => VideoChannelModel.unscoped(),
- required: false
+ required: false,
+ include: [
+ {
+ model: () => AccountModel,
+ required: true
+ }
+ ]
},
{
model: () => ServerModel,
uuid: this.uuid,
name: this.preferredUsername,
host: this.getHost(),
+ hostRedundancyAllowed: this.getRedundancyAllowed(),
followingCount: this.followingCount,
followersCount: this.followersCount,
avatar,
return this.Server ? this.Server.host : CONFIG.WEBSERVER.HOST
}
+ getRedundancyAllowed () {
+ return this.Server ? this.Server.redundancyAllowed : false
+ }
+
getAvatarUrl () {
if (!this.avatarId) return undefined
--- /dev/null
+import {
+ AfterDestroy,
+ AllowNull,
+ BelongsTo,
+ Column,
+ CreatedAt,
+ DataType,
+ ForeignKey,
+ Is,
+ Model,
+ Scopes,
+ Sequelize,
+ Table,
+ UpdatedAt
+} from 'sequelize-typescript'
+import { ActorModel } from '../activitypub/actor'
+import { throwIfNotValid } from '../utils'
+import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
+import { CONSTRAINTS_FIELDS, VIDEO_EXT_MIMETYPE } from '../../initializers'
+import { VideoFileModel } from '../video/video-file'
+import { isDateValid } from '../../helpers/custom-validators/misc'
+import { getServerActor } from '../../helpers/utils'
+import { VideoModel } from '../video/video'
+import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
+import { logger } from '../../helpers/logger'
+import { CacheFileObject } from '../../../shared'
+import { VideoChannelModel } from '../video/video-channel'
+import { ServerModel } from '../server/server'
+import { sample } from 'lodash'
+import { isTestInstance } from '../../helpers/core-utils'
+
+export enum ScopeNames {
+ WITH_VIDEO = 'WITH_VIDEO'
+}
+
+@Scopes({
+ [ ScopeNames.WITH_VIDEO ]: {
+ include: [
+ {
+ model: () => VideoFileModel,
+ required: true,
+ include: [
+ {
+ model: () => VideoModel,
+ required: true
+ }
+ ]
+ }
+ ]
+ }
+})
+
+@Table({
+ tableName: 'videoRedundancy',
+ indexes: [
+ {
+ fields: [ 'videoFileId' ]
+ },
+ {
+ fields: [ 'actorId' ]
+ },
+ {
+ fields: [ 'url' ],
+ unique: true
+ }
+ ]
+})
+export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
+
+ @CreatedAt
+ createdAt: Date
+
+ @UpdatedAt
+ updatedAt: Date
+
+ @AllowNull(false)
+ @Column
+ expiresOn: Date
+
+ @AllowNull(false)
+ @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
+ @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
+ fileUrl: string
+
+ @AllowNull(false)
+ @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
+ @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
+ url: string
+
+ @AllowNull(true)
+ @Column
+ strategy: string // Only used by us
+
+ @ForeignKey(() => VideoFileModel)
+ @Column
+ videoFileId: number
+
+ @BelongsTo(() => VideoFileModel, {
+ foreignKey: {
+ allowNull: false
+ },
+ onDelete: 'cascade'
+ })
+ VideoFile: VideoFileModel
+
+ @ForeignKey(() => ActorModel)
+ @Column
+ actorId: number
+
+ @BelongsTo(() => ActorModel, {
+ foreignKey: {
+ allowNull: false
+ },
+ onDelete: 'cascade'
+ })
+ Actor: ActorModel
+
+ @AfterDestroy
+ static removeFilesAndSendDelete (instance: VideoRedundancyModel) {
+ // Not us
+ if (!instance.strategy) return
+
+ logger.info('Removing video file %s-.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution)
+
+ return instance.VideoFile.Video.removeFile(instance.VideoFile)
+ }
+
+ static loadByFileId (videoFileId: number) {
+ const query = {
+ where: {
+ videoFileId
+ }
+ }
+
+ return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
+ }
+
+ static loadByUrl (url: string) {
+ const query = {
+ where: {
+ url
+ }
+ }
+
+ return VideoRedundancyModel.findOne(query)
+ }
+
+ static async findMostViewToDuplicate (randomizedFactor: number) {
+ // On VideoModel!
+ const query = {
+ logging: !isTestInstance(),
+ limit: randomizedFactor,
+ order: [ [ 'views', 'DESC' ] ],
+ include: [
+ {
+ model: VideoFileModel.unscoped(),
+ required: true,
+ where: {
+ id: {
+ [ Sequelize.Op.notIn ]: await VideoRedundancyModel.buildExcludeIn()
+ }
+ }
+ },
+ {
+ attributes: [],
+ model: VideoChannelModel.unscoped(),
+ required: true,
+ include: [
+ {
+ attributes: [],
+ model: ActorModel.unscoped(),
+ required: true,
+ include: [
+ {
+ attributes: [],
+ model: ServerModel.unscoped(),
+ required: true,
+ where: {
+ redundancyAllowed: true
+ }
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+
+ const rows = await VideoModel.unscoped().findAll(query)
+
+ return sample(rows)
+ }
+
+ static async getVideoFiles (strategy: VideoRedundancyStrategy) {
+ const actor = await getServerActor()
+
+ const queryVideoFiles = {
+ logging: !isTestInstance(),
+ where: {
+ actorId: actor.id,
+ strategy
+ }
+ }
+
+ return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO)
+ .findAll(queryVideoFiles)
+ }
+
+ static listAllExpired () {
+ const query = {
+ logging: !isTestInstance(),
+ where: {
+ expiresOn: {
+ [Sequelize.Op.lt]: new Date()
+ }
+ }
+ }
+
+ return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO)
+ .findAll(query)
+ }
+
+ toActivityPubObject (): CacheFileObject {
+ return {
+ id: this.url,
+ type: 'CacheFile' as 'CacheFile',
+ object: this.VideoFile.Video.url,
+ expires: this.expiresOn.toISOString(),
+ url: {
+ type: 'Link',
+ mimeType: VIDEO_EXT_MIMETYPE[ this.VideoFile.extname ] as any,
+ href: this.fileUrl,
+ height: this.VideoFile.resolution,
+ size: this.VideoFile.size,
+ fps: this.VideoFile.fps
+ }
+ }
+ }
+
+ private static async buildExcludeIn () {
+ const actor = await getServerActor()
+
+ return Sequelize.literal(
+ '(' +
+ `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` +
+ ')'
+ )
+ }
+}
-import { AllowNull, Column, CreatedAt, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import { AllowNull, Column, CreatedAt, Default, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
import { isHostValid } from '../../helpers/custom-validators/servers'
import { ActorModel } from '../activitypub/actor'
import { throwIfNotValid } from '../utils'
@Column
host: string
+ @AllowNull(false)
+ @Default(false)
+ @Column
+ redundancyAllowed: boolean
+
@CreatedAt
createdAt: Date
hooks: true
})
Actors: ActorModel[]
+
+ static loadByHost (host: string) {
+ const query = {
+ where: {
+ host
+ }
+ }
+
+ return ServerModel.findOne(query)
+ }
}
import { values } from 'lodash'
-import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript'
+import {
+ AllowNull,
+ BelongsTo,
+ Column,
+ CreatedAt,
+ DataType,
+ Default,
+ ForeignKey,
+ HasMany,
+ Is,
+ Model,
+ Table,
+ UpdatedAt
+} from 'sequelize-typescript'
import {
isVideoFileInfoHashValid,
isVideoFileResolutionValid,
import { throwIfNotValid } from '../utils'
import { VideoModel } from './video'
import * as Sequelize from 'sequelize'
+import { VideoRedundancyModel } from '../redundancy/video-redundancy'
@Table({
tableName: 'videoFile',
})
Video: VideoModel
+ @HasMany(() => VideoRedundancyModel, {
+ foreignKey: {
+ allowNull: false
+ },
+ onDelete: 'CASCADE',
+ hooks: true
+ })
+ RedundancyVideos: VideoRedundancyModel[]
+
static isInfohashExists (infoHash: string) {
const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
const options = {
Table,
UpdatedAt
} from 'sequelize-typescript'
-import { VideoPrivacy, VideoResolution, VideoState } from '../../../shared'
+import { ActivityUrlObject, VideoPrivacy, VideoResolution, VideoState } from '../../../shared'
import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos'
import { VideoFilter } from '../../../shared/models/videos/video-query.type'
import { createTorrentPromise, peertubeTruncate } from '../../helpers/core-utils'
import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
-import { isBooleanValid } from '../../helpers/custom-validators/misc'
+import { isArray, isBooleanValid } from '../../helpers/custom-validators/misc'
import {
isVideoCategoryValid,
isVideoDescriptionValid,
import { VideoBlacklistModel } from './video-blacklist'
import { copy, remove, rename, stat, writeFile } from 'fs-extra'
import { VideoViewModel } from './video-views'
+import { VideoRedundancyModel } from '../redundancy/video-redundancy'
// FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation
const indexes: Sequelize.DefineIndexesOptions[] = [
include: [
{
model: () => VideoFileModel.unscoped(),
- required: false
+ required: false,
+ include: [
+ {
+ model: () => VideoRedundancyModel.unscoped(),
+ required: false
+ }
+ ]
}
]
},
name: 'videoId',
allowNull: false
},
+ hooks: true,
onDelete: 'cascade'
})
VideoFiles: VideoFileModel[]
[ CONFIG.WEBSERVER.WS + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT + '/tracker/socket' ],
[ CONFIG.WEBSERVER.URL + '/tracker/announce' ]
],
- urlList: [
- CONFIG.WEBSERVER.URL + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile)
- ]
+ urlList: [ CONFIG.WEBSERVER.URL + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile) ]
}
const torrent = await createTorrentPromise(this.getVideoFilePath(videoFile), options)
}
}
- const url = []
+ const url: ActivityUrlObject[] = []
for (const file of this.VideoFiles) {
url.push({
type: 'Link',
- mimeType: VIDEO_EXT_MIMETYPE[ file.extname ],
+ mimeType: VIDEO_EXT_MIMETYPE[ file.extname ] as any,
href: this.getVideoFileUrl(file, baseUrlHttp),
height: file.resolution,
size: file.size,
url.push({
type: 'Link',
- mimeType: 'application/x-bittorrent',
+ mimeType: 'application/x-bittorrent' as 'application/x-bittorrent',
href: this.getTorrentUrl(file, baseUrlHttp),
height: file.resolution
})
url.push({
type: 'Link',
- mimeType: 'application/x-bittorrent;x-scheme-handler/magnet',
+ mimeType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet',
href: this.generateMagnetUri(file, baseUrlHttp, baseUrlWs),
height: file.resolution
})
(now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL
}
- private getBaseUrls () {
+ getBaseUrls () {
let baseUrlHttp
let baseUrlWs
return { baseUrlHttp, baseUrlWs }
}
- private getThumbnailUrl (baseUrlHttp: string) {
+ generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) {
+ const xs = this.getTorrentUrl(videoFile, baseUrlHttp)
+ const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ]
+ let urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ]
+
+ const redundancies = videoFile.RedundancyVideos
+ if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl))
+
+ const magnetHash = {
+ xs,
+ announce,
+ urlList,
+ infoHash: videoFile.infoHash,
+ name: this.name
+ }
+
+ return magnetUtil.encode(magnetHash)
+ }
+
+ getThumbnailUrl (baseUrlHttp: string) {
return baseUrlHttp + STATIC_PATHS.THUMBNAILS + this.getThumbnailName()
}
- private getTorrentUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
+ getTorrentUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
return baseUrlHttp + STATIC_PATHS.TORRENTS + this.getTorrentFileName(videoFile)
}
- private getTorrentDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
+ getTorrentDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
return baseUrlHttp + STATIC_DOWNLOAD_PATHS.TORRENTS + this.getTorrentFileName(videoFile)
}
- private getVideoFileUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
+ getVideoFileUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
return baseUrlHttp + STATIC_PATHS.WEBSEED + this.getVideoFilename(videoFile)
}
- private getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
+ getVideoFileDownloadUrl (videoFile: VideoFileModel, baseUrlHttp: string) {
return baseUrlHttp + STATIC_DOWNLOAD_PATHS.VIDEOS + this.getVideoFilename(videoFile)
}
-
- private generateMagnetUri (videoFile: VideoFileModel, baseUrlHttp: string, baseUrlWs: string) {
- const xs = this.getTorrentUrl(videoFile, baseUrlHttp)
- const announce = [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ]
- const urlList = [ this.getVideoFileUrl(videoFile, baseUrlHttp) ]
-
- const magnetHash = {
- xs,
- announce,
- urlList,
- infoHash: videoFile.infoHash,
- name: this.name
- }
-
- return magnetUtil.encode(magnetHash)
- }
}
statusCodeExpected: 404
})
})
-
- it('Should succeed with the correct parameters', async function () {
- await makeDeleteRequest({
- url: server.url,
- path: path + '/localhost:9002',
- token: server.accessToken,
- statusCodeExpected: 404
- })
- })
})
})
// Order of the tests we want to execute
import './accounts'
+import './config'
import './follows'
import './jobs'
+import './redundancy'
+import './search'
import './services'
+import './user-subscriptions'
import './users'
import './video-abuses'
import './video-blacklist'
import './video-captions'
import './video-channels'
import './video-comments'
-import './videos'
import './video-imports'
-import './search'
-import './user-subscriptions'
+import './videos'
--- /dev/null
+/* tslint:disable:no-unused-expression */
+
+import 'mocha'
+
+import {
+ createUser,
+ doubleFollow,
+ flushAndRunMultipleServers,
+ flushTests,
+ killallServers,
+ makePutBodyRequest,
+ ServerInfo,
+ setAccessTokensToServers,
+ userLogin
+} from '../../utils'
+
+describe('Test server redundancy API validators', function () {
+ let servers: ServerInfo[]
+ let userAccessToken = null
+
+ // ---------------------------------------------------------------
+
+ before(async function () {
+ this.timeout(30000)
+
+ await flushTests()
+ servers = await flushAndRunMultipleServers(2)
+
+ await setAccessTokensToServers(servers)
+ await doubleFollow(servers[0], servers[1])
+
+ const user = {
+ username: 'user1',
+ password: 'password'
+ }
+
+ await createUser(servers[0].url, servers[0].accessToken, user.username, user.password)
+ userAccessToken = await userLogin(servers[0], user)
+ })
+
+ describe('When updating redundancy', function () {
+ const path = '/api/v1/server/redundancy'
+
+ it('Should fail with an invalid token', async function () {
+ await makePutBodyRequest({
+ url: servers[0].url,
+ path: path + '/localhost:9002',
+ fields: { redundancyAllowed: true },
+ token: 'fake_token',
+ statusCodeExpected: 401
+ })
+ })
+
+ it('Should fail if the user is not an administrator', async function () {
+ await makePutBodyRequest({
+ url: servers[0].url,
+ path: path + '/localhost:9002',
+ fields: { redundancyAllowed: true },
+ token: userAccessToken,
+ statusCodeExpected: 403
+ })
+ })
+
+ it('Should fail if we do not follow this server', async function () {
+ await makePutBodyRequest({
+ url: servers[0].url,
+ path: path + '/example.com',
+ fields: { redundancyAllowed: true },
+ token: servers[0].accessToken,
+ statusCodeExpected: 404
+ })
+ })
+
+ it('Should fail without de redundancyAllowed param', async function () {
+ await makePutBodyRequest({
+ url: servers[0].url,
+ path: path + '/localhost:9002',
+ fields: { blabla: true },
+ token: servers[0].accessToken,
+ statusCodeExpected: 400
+ })
+ })
+
+ it('Should succeed with the correct parameters', async function () {
+ await makePutBodyRequest({
+ url: servers[0].url,
+ path: path + '/localhost:9002',
+ fields: { redundancyAllowed: true },
+ token: servers[0].accessToken,
+ statusCodeExpected: 204
+ })
+ })
+ })
+
+ after(async function () {
+ killallServers(servers)
+
+ // Keep the logs if the test failed
+ if (this['ok']) {
+ await flushTests()
+ }
+ })
+})
import './follows'
import './handle-down'
import './jobs'
+import './redundancy'
import './reverse-proxy'
import './stats'
import './tracker'
--- /dev/null
+/* tslint:disable:no-unused-expression */
+
+import * as chai from 'chai'
+import 'mocha'
+import { VideoDetails } from '../../../../shared/models/videos'
+import {
+ doubleFollow,
+ flushAndRunMultipleServers,
+ flushTests,
+ getFollowingListPaginationAndSort,
+ getVideo,
+ killallServers,
+ ServerInfo,
+ setAccessTokensToServers,
+ uploadVideo,
+ wait,
+ root, viewVideo
+} from '../../utils'
+import { waitJobs } from '../../utils/server/jobs'
+import * as magnetUtil from 'magnet-uri'
+import { updateRedundancy } from '../../utils/server/redundancy'
+import { ActorFollow } from '../../../../shared/models/actors'
+import { readdir } from 'fs-extra'
+import { join } from 'path'
+
+const expect = chai.expect
+
+function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) {
+ const parsed = magnetUtil.decode(file.magnetUri)
+
+ for (const ws of baseWebseeds) {
+ const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
+ expect(found, `Webseed ${ws} not found in ${file.magnetUri}`).to.not.be.undefined
+ }
+}
+
+describe('Test videos redundancy', function () {
+ let servers: ServerInfo[] = []
+ let video1Server2UUID: string
+ let video2Server2UUID: string
+
+ before(async function () {
+ this.timeout(120000)
+
+ servers = await flushAndRunMultipleServers(3)
+
+ // Get the access tokens
+ await setAccessTokensToServers(servers)
+
+ {
+ const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
+ video1Server2UUID = res.body.video.uuid
+
+ await viewVideo(servers[1].url, video1Server2UUID)
+ }
+
+ {
+ const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
+ video2Server2UUID = res.body.video.uuid
+ }
+
+ await waitJobs(servers)
+
+ // Server 1 and server 2 follow each other
+ await doubleFollow(servers[0], servers[1])
+ // Server 1 and server 3 follow each other
+ await doubleFollow(servers[0], servers[2])
+ // Server 2 and server 3 follow each other
+ await doubleFollow(servers[1], servers[2])
+
+ await waitJobs(servers)
+ })
+
+ it('Should have 1 webseed on the first video', async function () {
+ const webseeds = [
+ 'http://localhost:9002/static/webseed/' + video1Server2UUID
+ ]
+
+ for (const server of servers) {
+ const res = await getVideo(server.url, video1Server2UUID)
+
+ const video: VideoDetails = res.body
+ video.files.forEach(f => checkMagnetWebseeds(f, webseeds))
+ }
+ })
+
+ it('Should enable redundancy on server 1', async function () {
+ await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true)
+
+ const res = await getFollowingListPaginationAndSort(servers[0].url, 0, 5, '-createdAt')
+ const follows: ActorFollow[] = res.body.data
+ const server2 = follows.find(f => f.following.host === 'localhost:9002')
+ const server3 = follows.find(f => f.following.host === 'localhost:9003')
+
+ expect(server3).to.not.be.undefined
+ expect(server3.following.hostRedundancyAllowed).to.be.false
+
+ expect(server2).to.not.be.undefined
+ expect(server2.following.hostRedundancyAllowed).to.be.true
+ })
+
+ it('Should have 2 webseed on the first video', async function () {
+ this.timeout(40000)
+
+ await waitJobs(servers)
+ await wait(15000)
+ await waitJobs(servers)
+
+ const webseeds = [
+ 'http://localhost:9001/static/webseed/' + video1Server2UUID,
+ 'http://localhost:9002/static/webseed/' + video1Server2UUID
+ ]
+
+ for (const server of servers) {
+ const res = await getVideo(server.url, video1Server2UUID)
+
+ const video: VideoDetails = res.body
+
+ for (const file of video.files) {
+ checkMagnetWebseeds(file, webseeds)
+ }
+ }
+
+ const files = await readdir(join(root(), 'test1', 'videos'))
+ expect(files).to.have.lengthOf(4)
+
+ for (const resolution of [ 240, 360, 480, 720 ]) {
+ expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined
+ }
+ })
+
+ after(async function () {
+ killallServers(servers)
+
+ // Keep the logs if the test failed
+ if (this['ok']) {
+ await flushTests()
+ }
+ })
+})
import * as request from 'supertest'
-import { wait } from '../miscs/miscs'
import { ServerInfo } from './servers'
import { waitJobs } from './jobs'
--- /dev/null
+import { makePutBodyRequest } from '../requests/requests'
+
+async function updateRedundancy (url: string, accessToken: string, host: string, redundancyAllowed: boolean, expectedStatus = 204) {
+ const path = '/api/v1/server/redundancy/' + host
+
+ return makePutBodyRequest({
+ url,
+ path,
+ token: accessToken,
+ fields: { redundancyAllowed },
+ statusCodeExpected: expectedStatus
+ })
+}
+
+export {
+ updateRedundancy
+}
import { ActivityPubActor } from './activitypub-actor'
import { ActivityPubSignature } from './activitypub-signature'
-import { VideoTorrentObject } from './objects'
+import { CacheFileObject, VideoTorrentObject } from './objects'
import { DislikeObject } from './objects/dislike-object'
import { VideoAbuseObject } from './objects/video-abuse-object'
import { VideoCommentObject } from './objects/video-comment-object'
export interface ActivityCreate extends BaseActivity {
type: 'Create'
- object: VideoTorrentObject | VideoAbuseObject | ViewObject | DislikeObject | VideoCommentObject
+ object: VideoTorrentObject | VideoAbuseObject | ViewObject | DislikeObject | VideoCommentObject | CacheFileObject
}
export interface ActivityUpdate extends BaseActivity {
type: 'Update'
- object: VideoTorrentObject | ActivityPubActor
+ object: VideoTorrentObject | ActivityPubActor | CacheFileObject
}
export interface ActivityDelete extends BaseActivity {
--- /dev/null
+import { ActivityVideoUrlObject } from './common-objects'
+
+export interface CacheFileObject {
+ id: string
+ type: 'CacheFile',
+ object: string
+ expires: string
+ url: ActivityVideoUrlObject
+}
height: number
}
-export interface ActivityUrlObject {
+export type ActivityVideoUrlObject = {
type: 'Link'
- mimeType: 'video/mp4' | 'video/webm' | 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet'
+ mimeType: 'video/mp4' | 'video/webm' | 'video/ogg'
href: string
height: number
-
- size?: number
- fps?: number
+ size: number
+ fps: number
}
+export type ActivityUrlObject =
+ ActivityVideoUrlObject
+ |
+ {
+ type: 'Link'
+ mimeType: 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet'
+ href: string
+ height: number
+ }
+ |
+ {
+ type: 'Link'
+ mimeType: 'text/html'
+ href: string
+ }
+
export interface ActivityPubAttributedTo {
type: 'Group' | 'Person'
id: string
+export * from './cache-file-object'
export * from './common-objects'
export * from './video-abuse-object'
export * from './video-torrent-object'
import {
ActivityIconObject,
- ActivityIdentifierObject, ActivityPubAttributedTo,
+ ActivityIdentifierObject,
+ ActivityPubAttributedTo,
ActivityTagObject,
ActivityUrlObject
} from './common-objects'
-import { ActivityPubOrderedCollection } from '../activitypub-ordered-collection'
import { VideoState } from '../../videos'
export interface VideoTorrentObject {
export type FollowState = 'pending' | 'accepted'
-export interface AccountFollow {
+export interface ActorFollow {
id: number
- follower: Actor
- following: Actor
+ follower: Actor & { hostRedundancyAllowed: boolean }
+ following: Actor & { hostRedundancyAllowed: boolean }
score: number
state: FollowState
createdAt: Date
--- /dev/null
+export * from './avatar.model'
-export * from './actors'
export * from './activitypub'
+export * from './actors'
+export * from './avatars'
+export * from './redundancy'
export * from './users'
export * from './videos'
export * from './feeds'
--- /dev/null
+export * from './videos-redundancy.model'
--- /dev/null
+export type VideoRedundancyStrategy = 'most-views'
+
+export interface VideosRedundancy {
+ strategy: VideoRedundancyStrategy
+ size: number
+}
MANAGE_USERS,
MANAGE_SERVER_FOLLOW,
+ MANAGE_SERVER_REDUNDANCY,
MANAGE_VIDEO_ABUSES,
MANAGE_JOBS,
MANAGE_CONFIGURATION,
"@types/bluebird" "*"
"@types/ioredis" "*"
+"@types/bytes@^3.0.0":
+ version "3.0.0"
+ resolved "https://registry.yarnpkg.com/@types/bytes/-/bytes-3.0.0.tgz#549eeacd0a8fecfaa459334583a4edcee738e6db"
+
"@types/caseless@*":
version "0.12.1"
resolved "https://registry.yarnpkg.com/@types/caseless/-/caseless-0.12.1.tgz#9794c69c8385d0192acc471a540d1f8e0d16218a"
version "1.0.0"
resolved "https://registry.yarnpkg.com/bytes/-/bytes-1.0.0.tgz#3569ede8ba34315fab99c3e92cb04c7220de1fa8"
-bytes@3.0.0:
+bytes@3.0.0, bytes@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.0.0.tgz#d32815404d689699f85a4ea4fa8755dd13a96048"
projects / github / Chocobozzz / PeerTube.git / commitdiff
