aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2020-01-10 10:11:28 +0100
committerChocobozzz <chocobozzz@cpy.re>2020-01-28 11:35:26 +0100
commitb764380ac23f4e9d4677d08acdc3474c2931a16d (patch)
tree0d2c440ed8b56c35e47f2274586a11da63852086
parent3ae0bbd23c6f1b2790975328d8eae6a8317c223d (diff)
downloadPeerTube-b764380ac23f4e9d4677d08acdc3474c2931a16d.tar.gz
PeerTube-b764380ac23f4e9d4677d08acdc3474c2931a16d.tar.zst
PeerTube-b764380ac23f4e9d4677d08acdc3474c2931a16d.zip
Add ability to list redundancies
-rw-r--r--client/package.json1
-rw-r--r--client/src/app/+admin/admin.component.html2
-rw-r--r--client/src/app/+admin/admin.module.ts13
-rw-r--r--client/src/app/+admin/follows/follows.component.html6
-rw-r--r--client/src/app/+admin/follows/follows.routes.ts5
-rw-r--r--client/src/app/+admin/follows/index.ts1
-rw-r--r--client/src/app/+admin/follows/shared/redundancy-checkbox.component.ts2
-rw-r--r--client/src/app/+admin/follows/shared/redundancy.service.ts28
-rw-r--r--client/src/app/+admin/follows/video-redundancies-list/index.ts1
-rw-r--r--client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.html82
-rw-r--r--client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.scss37
-rw-r--r--client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts178
-rw-r--r--client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.html24
-rw-r--r--client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.scss8
-rw-r--r--client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts11
-rw-r--r--client/src/app/+admin/system/jobs/jobs.component.ts15
-rw-r--r--client/src/app/core/server/server.service.ts7
-rw-r--r--client/src/app/shared/images/global-icon.component.ts1
-rw-r--r--client/src/app/shared/instance/instance-statistics.component.ts23
-rw-r--r--client/src/app/shared/shared.module.ts2
-rw-r--r--client/src/app/shared/video/redundancy.service.ts73
-rw-r--r--client/src/app/shared/video/video-actions-dropdown.component.ts28
-rw-r--r--client/src/app/shared/video/video-miniature.component.ts3
-rw-r--r--client/src/app/shared/video/video.model.ts4
-rw-r--r--client/yarn.lock35
-rw-r--r--config/test.yaml8
-rw-r--r--package.json2
-rwxr-xr-xscripts/dev/index.sh4
-rw-r--r--server/controllers/api/server/follows.ts4
-rw-r--r--server/controllers/api/server/redundancy.ts84
-rw-r--r--server/controllers/api/server/stats.ts10
-rw-r--r--server/helpers/custom-validators/activitypub/cache-file.ts2
-rw-r--r--server/helpers/custom-validators/video-redundancies.ts12
-rw-r--r--server/helpers/webtorrent.ts4
-rw-r--r--server/initializers/config.ts4
-rw-r--r--server/initializers/constants.ts25
-rw-r--r--server/initializers/migrations/0475-redundancy-expires-on.ts27
-rw-r--r--server/lib/activitypub/cache-file.ts4
-rw-r--r--server/lib/job-queue/handlers/video-redundancy.ts20
-rw-r--r--server/lib/job-queue/job-queue.ts13
-rw-r--r--server/lib/redundancy.ts8
-rw-r--r--server/lib/schedulers/update-videos-scheduler.ts1
-rw-r--r--server/lib/schedulers/videos-redundancy-scheduler.ts59
-rw-r--r--server/middlewares/sort.ts23
-rw-r--r--server/middlewares/validators/redundancy.ts74
-rw-r--r--server/middlewares/validators/sort.ts3
-rw-r--r--server/models/redundancy/video-redundancy.ts186
-rw-r--r--server/tests/api/check-params/redundancy.ts141
-rw-r--r--server/tests/api/redundancy/index.ts1
-rw-r--r--server/tests/api/redundancy/manage-redundancy.ts373
-rw-r--r--server/tests/api/redundancy/redundancy.ts138
-rw-r--r--server/typings/models/video/video-file.ts5
-rw-r--r--server/typings/models/video/video-streaming-playlist.ts6
-rw-r--r--server/typings/models/video/video.ts13
-rw-r--r--shared/extra-utils/server/redundancy.ts70
-rw-r--r--shared/extra-utils/videos/videos.ts18
-rw-r--r--shared/models/redundancy/index.ts4
-rw-r--r--shared/models/redundancy/video-redundancies-filters.model.ts1
-rw-r--r--shared/models/redundancy/video-redundancy.model.ts33
-rw-r--r--shared/models/redundancy/videos-redundancy-strategy.model.ts (renamed from shared/models/redundancy/videos-redundancy.model.ts)3
-rw-r--r--shared/models/server/job.model.ts3
-rw-r--r--shared/models/server/server-stats.model.ts18
-rw-r--r--shared/models/users/user-right.enum.ts4
-rw-r--r--shared/models/videos/video.model.ts2
64 files changed, 1806 insertions, 194 deletions
diff --git a/client/package.json b/client/package.json
index cd0a82aa4..7205dbe8f 100644
--- a/client/package.json
+++ b/client/package.json
@@ -77,6 +77,7 @@
77 "bootstrap": "^4.1.3", 77 "bootstrap": "^4.1.3",
78 "buffer": "^5.1.0", 78 "buffer": "^5.1.0",
79 "cache-chunk-store": "^3.0.0", 79 "cache-chunk-store": "^3.0.0",
80 "chart.js": "^2.9.3",
80 "codelyzer": "^5.0.1", 81 "codelyzer": "^5.0.1",
81 "core-js": "^3.1.4", 82 "core-js": "^3.1.4",
82 "css-loader": "^3.1.0", 83 "css-loader": "^3.1.0",
diff --git a/client/src/app/+admin/admin.component.html b/client/src/app/+admin/admin.component.html
index 9a3d90c18..0d06aaedc 100644
--- a/client/src/app/+admin/admin.component.html
+++ b/client/src/app/+admin/admin.component.html
@@ -5,7 +5,7 @@
5 </a> 5 </a>
6 6
7 <a i18n *ngIf="hasServerFollowRight()" routerLink="/admin/follows" routerLinkActive="active" class="title-page"> 7 <a i18n *ngIf="hasServerFollowRight()" routerLink="/admin/follows" routerLinkActive="active" class="title-page">
8 Manage follows 8 Follows & redundancies
9 </a> 9 </a>
10 10
11 <a i18n *ngIf="hasVideoAbusesRight() || hasVideoBlacklistRight()" routerLink="/admin/moderation" routerLinkActive="active" class="title-page"> 11 <a i18n *ngIf="hasVideoAbusesRight() || hasVideoBlacklistRight()" routerLink="/admin/moderation" routerLinkActive="active" class="title-page">
diff --git a/client/src/app/+admin/admin.module.ts b/client/src/app/+admin/admin.module.ts
index 9c56b5750..fdbe70314 100644
--- a/client/src/app/+admin/admin.module.ts
+++ b/client/src/app/+admin/admin.module.ts
@@ -5,7 +5,7 @@ import { TableModule } from 'primeng/table'
5import { SharedModule } from '../shared' 5import { SharedModule } from '../shared'
6import { AdminRoutingModule } from './admin-routing.module' 6import { AdminRoutingModule } from './admin-routing.module'
7import { AdminComponent } from './admin.component' 7import { AdminComponent } from './admin.component'
8import { FollowersListComponent, FollowingAddComponent, FollowsComponent } from './follows' 8import { FollowersListComponent, FollowingAddComponent, FollowsComponent, VideoRedundanciesListComponent } from './follows'
9import { FollowingListComponent } from './follows/following-list/following-list.component' 9import { FollowingListComponent } from './follows/following-list/following-list.component'
10import { UserCreateComponent, UserListComponent, UserPasswordComponent, UsersComponent, UserUpdateComponent } from './users' 10import { UserCreateComponent, UserListComponent, UserPasswordComponent, UsersComponent, UserUpdateComponent } from './users'
11import { 11import {
@@ -16,7 +16,6 @@ import {
16} from './moderation' 16} from './moderation'
17import { ModerationComponent } from '@app/+admin/moderation/moderation.component' 17import { ModerationComponent } from '@app/+admin/moderation/moderation.component'
18import { RedundancyCheckboxComponent } from '@app/+admin/follows/shared/redundancy-checkbox.component' 18import { RedundancyCheckboxComponent } from '@app/+admin/follows/shared/redundancy-checkbox.component'
19import { RedundancyService } from '@app/+admin/follows/shared/redundancy.service'
20import { InstanceAccountBlocklistComponent, InstanceServerBlocklistComponent } from '@app/+admin/moderation/instance-blocklist' 19import { InstanceAccountBlocklistComponent, InstanceServerBlocklistComponent } from '@app/+admin/moderation/instance-blocklist'
21import { JobsComponent } from '@app/+admin/system/jobs/jobs.component' 20import { JobsComponent } from '@app/+admin/system/jobs/jobs.component'
22import { JobService, LogsComponent, LogsService, SystemComponent } from '@app/+admin/system' 21import { JobService, LogsComponent, LogsService, SystemComponent } from '@app/+admin/system'
@@ -27,13 +26,18 @@ import { PluginSearchComponent } from '@app/+admin/plugins/plugin-search/plugin-
27import { PluginShowInstalledComponent } from '@app/+admin/plugins/plugin-show-installed/plugin-show-installed.component' 26import { PluginShowInstalledComponent } from '@app/+admin/plugins/plugin-show-installed/plugin-show-installed.component'
28import { SelectButtonModule } from 'primeng/selectbutton' 27import { SelectButtonModule } from 'primeng/selectbutton'
29import { PluginApiService } from '@app/+admin/plugins/shared/plugin-api.service' 28import { PluginApiService } from '@app/+admin/plugins/shared/plugin-api.service'
29import { VideoRedundancyInformationComponent } from '@app/+admin/follows/video-redundancies-list/video-redundancy-information.component'
30import { ChartModule } from 'primeng/chart'
30 31
31@NgModule({ 32@NgModule({
32 imports: [ 33 imports: [
33 AdminRoutingModule, 34 AdminRoutingModule,
35
36 SharedModule,
37
34 TableModule, 38 TableModule,
35 SelectButtonModule, 39 SelectButtonModule,
36 SharedModule 40 ChartModule
37 ], 41 ],
38 42
39 declarations: [ 43 declarations: [
@@ -44,6 +48,8 @@ import { PluginApiService } from '@app/+admin/plugins/shared/plugin-api.service'
44 FollowersListComponent, 48 FollowersListComponent,
45 FollowingListComponent, 49 FollowingListComponent,
46 RedundancyCheckboxComponent, 50 RedundancyCheckboxComponent,
51 VideoRedundanciesListComponent,
52 VideoRedundancyInformationComponent,
47 53
48 UsersComponent, 54 UsersComponent,
49 UserCreateComponent, 55 UserCreateComponent,
@@ -78,7 +84,6 @@ import { PluginApiService } from '@app/+admin/plugins/shared/plugin-api.service'
78 ], 84 ],
79 85
80 providers: [ 86 providers: [
81 RedundancyService,
82 JobService, 87 JobService,
83 LogsService, 88 LogsService,
84 DebugService, 89 DebugService,
diff --git a/client/src/app/+admin/follows/follows.component.html b/client/src/app/+admin/follows/follows.component.html
index 21d477132..46581daf9 100644
--- a/client/src/app/+admin/follows/follows.component.html
+++ b/client/src/app/+admin/follows/follows.component.html
@@ -1,5 +1,5 @@
1<div class="admin-sub-header"> 1<div class="admin-sub-header">
2 <div i18n class="form-sub-title">Manage follows</div> 2 <div i18n class="form-sub-title">Follows & redundancies</div>
3 3
4 <div class="admin-sub-nav"> 4 <div class="admin-sub-nav">
5 <a i18n routerLink="following-list" routerLinkActive="active">Following</a> 5 <a i18n routerLink="following-list" routerLinkActive="active">Following</a>
@@ -7,7 +7,9 @@
7 <a i18n routerLink="following-add" routerLinkActive="active">Follow</a> 7 <a i18n routerLink="following-add" routerLinkActive="active">Follow</a>
8 8
9 <a i18n routerLink="followers-list" routerLinkActive="active">Followers</a> 9 <a i18n routerLink="followers-list" routerLinkActive="active">Followers</a>
10
11 <a i18n routerLink="video-redundancies-list" routerLinkActive="active">Video redundancies</a>
10 </div> 12 </div>
11</div> 13</div>
12 14
13<router-outlet></router-outlet> \ No newline at end of file 15<router-outlet></router-outlet>
diff --git a/client/src/app/+admin/follows/follows.routes.ts b/client/src/app/+admin/follows/follows.routes.ts
index e84c79e82..298733eb0 100644
--- a/client/src/app/+admin/follows/follows.routes.ts
+++ b/client/src/app/+admin/follows/follows.routes.ts
@@ -6,6 +6,7 @@ import { FollowingAddComponent } from './following-add'
6import { FollowersListComponent } from './followers-list' 6import { FollowersListComponent } from './followers-list'
7import { UserRight } from '../../../../../shared' 7import { UserRight } from '../../../../../shared'
8import { FollowingListComponent } from './following-list/following-list.component' 8import { FollowingListComponent } from './following-list/following-list.component'
9import { VideoRedundanciesListComponent } from '@app/+admin/follows/video-redundancies-list'
9 10
10export const FollowsRoutes: Routes = [ 11export const FollowsRoutes: Routes = [
11 { 12 {
@@ -47,6 +48,10 @@ export const FollowsRoutes: Routes = [
47 title: 'Add follow' 48 title: 'Add follow'
48 } 49 }
49 } 50 }
51 },
52 {
53 path: 'video-redundancies-list',
54 component: VideoRedundanciesListComponent
50 } 55 }
51 ] 56 ]
52 } 57 }
diff --git a/client/src/app/+admin/follows/index.ts b/client/src/app/+admin/follows/index.ts
index e94f33710..4fcb35cb1 100644
--- a/client/src/app/+admin/follows/index.ts
+++ b/client/src/app/+admin/follows/index.ts
@@ -1,5 +1,6 @@
1export * from './following-add' 1export * from './following-add'
2export * from './followers-list' 2export * from './followers-list'
3export * from './following-list' 3export * from './following-list'
4export * from './video-redundancies-list'
4export * from './follows.component' 5export * from './follows.component'
5export * from './follows.routes' 6export * from './follows.routes'
diff --git a/client/src/app/+admin/follows/shared/redundancy-checkbox.component.ts b/client/src/app/+admin/follows/shared/redundancy-checkbox.component.ts
index fa1da26bf..9d7883d97 100644
--- a/client/src/app/+admin/follows/shared/redundancy-checkbox.component.ts
+++ b/client/src/app/+admin/follows/shared/redundancy-checkbox.component.ts
@@ -1,7 +1,7 @@
1import { Component, Input } from '@angular/core' 1import { Component, Input } from '@angular/core'
2import { Notifier } from '@app/core' 2import { Notifier } from '@app/core'
3import { I18n } from '@ngx-translate/i18n-polyfill' 3import { I18n } from '@ngx-translate/i18n-polyfill'
4import { RedundancyService } from '@app/+admin/follows/shared/redundancy.service' 4import { RedundancyService } from '@app/shared/video/redundancy.service'
5 5
6@Component({ 6@Component({
7 selector: 'my-redundancy-checkbox', 7 selector: 'my-redundancy-checkbox',
diff --git a/client/src/app/+admin/follows/shared/redundancy.service.ts b/client/src/app/+admin/follows/shared/redundancy.service.ts
deleted file mode 100644
index 87ae01c04..000000000
--- a/client/src/app/+admin/follows/shared/redundancy.service.ts
+++ /dev/null
@@ -1,28 +0,0 @@
1import { catchError, map } from 'rxjs/operators'
2import { HttpClient } from '@angular/common/http'
3import { Injectable } from '@angular/core'
4import { RestExtractor } from '@app/shared'
5import { environment } from '../../../../environments/environment'
6
7@Injectable()
8export class RedundancyService {
9 static BASE_USER_SUBSCRIPTIONS_URL = environment.apiUrl + '/api/v1/server/redundancy'
10
11 constructor (
12 private authHttp: HttpClient,
13 private restExtractor: RestExtractor
14 ) { }
15
16 updateRedundancy (host: string, redundancyAllowed: boolean) {
17 const url = RedundancyService.BASE_USER_SUBSCRIPTIONS_URL + '/' + host
18
19 const body = { redundancyAllowed }
20
21 return this.authHttp.put(url, body)
22 .pipe(
23 map(this.restExtractor.extractDataBool),
24 catchError(err => this.restExtractor.handleError(err))
25 )
26 }
27
28}
diff --git a/client/src/app/+admin/follows/video-redundancies-list/index.ts b/client/src/app/+admin/follows/video-redundancies-list/index.ts
new file mode 100644
index 000000000..6a7c7f483
--- /dev/null
+++ b/client/src/app/+admin/follows/video-redundancies-list/index.ts
@@ -0,0 +1 @@
export * from './video-redundancies-list.component'
diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.html b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.html
new file mode 100644
index 000000000..80c66ec60
--- /dev/null
+++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.html
@@ -0,0 +1,82 @@
1<div class="admin-sub-header">
2 <div i18n class="form-sub-title">Video redundancies list</div>
3
4 <div class="select-filter-block">
5 <label for="displayType" i18n>Display</label>
6
7 <div class="peertube-select-container">
8 <select id="displayType" name="displayType" [(ngModel)]="displayType" (ngModelChange)="onDisplayTypeChanged()">
9 <option value="my-videos">My videos duplicated by remote instances</option>
10 <option value="remote-videos">Remote videos duplicated by my instance</option>
11 </select>
12 </div>
13 </div>
14</div>
15
16<p-table
17 [value]="videoRedundancies" [lazy]="true" [paginator]="true" [totalRecords]="totalRecords" [rows]="rowsPerPage"
18 [sortField]="sort.field" [sortOrder]="sort.order" (onLazyLoad)="loadLazy($event)" dataKey="id"
19>
20 <ng-template pTemplate="header">
21 <tr>
22 <th i18n *ngIf="isDisplayingRemoteVideos()">Strategy</th>
23 <th i18n pSortableColumn="name">Video name <p-sortIcon field="name"></p-sortIcon></th>
24 <th i18n>Video URL</th>
25 <th i18n *ngIf="isDisplayingRemoteVideos()">Total size</th>
26 <th></th>
27 </tr>
28 </ng-template>
29
30 <ng-template pTemplate="body" let-redundancy>
31 <tr class="expander" [pRowToggler]="redundancy">
32 <td *ngIf="isDisplayingRemoteVideos()">{{ getRedundancyStrategy(redundancy) }}</td>
33
34 <td>{{ redundancy.name }}</td>
35
36 <td>
37 <a target="_blank" rel="noopener noreferrer" [href]="redundancy.url">{{ redundancy.url }}</a>
38 </td>
39
40 <td *ngIf="isDisplayingRemoteVideos()">{{ getTotalSize(redundancy) | bytes: 1 }}</td>
41
42 <td class="action-cell">
43 <my-delete-button (click)="removeRedundancy(redundancy)"></my-delete-button>
44 </td>
45 </tr>
46 </ng-template>
47
48 <ng-template pTemplate="rowexpansion" let-redundancy>
49 <tr>
50 <td colspan="2">
51 <div *ngFor="let file of redundancy.redundancies.files" class="expansion-block">
52 <my-video-redundancy-information [redundancyElement]="file"></my-video-redundancy-information>
53 </div>
54 </td>
55 </tr>
56
57 <tr>
58 <td colspan="2">
59 <div *ngFor="let playlist of redundancy.redundancies.streamingPlaylists">
60 <my-video-redundancy-information [redundancyElement]="playlist"></my-video-redundancy-information>
61 </div>
62 </td>
63 </tr>
64 </ng-template>
65</p-table>
66
67
68<div class="redundancies-charts" *ngIf="isDisplayingRemoteVideos()">
69 <div class="form-sub-title" i18n>Enabled strategies stats</div>
70
71 <div class="chart-blocks">
72
73 <div *ngIf="noRedundancies" i18n class="no-results">
74 No redundancy strategy is enabled on your instance.
75 </div>
76
77 <div class="chart-block" *ngFor="let r of redundanciesGraphsData">
78 <p-chart type="pie" [data]="r.graphData" [options]="r.options" width="300px" height="300px"></p-chart>
79 </div>
80
81 </div>
82</div>
diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.scss b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.scss
new file mode 100644
index 000000000..05018c281
--- /dev/null
+++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.scss
@@ -0,0 +1,37 @@
1@import '_variables';
2@import '_mixins';
3
4.expansion-block {
5 margin-bottom: 20px;
6}
7
8.admin-sub-header {
9 align-items: flex-end;
10
11 .select-filter-block {
12 &:not(:last-child) {
13 margin-right: 10px;
14 }
15
16 label {
17 margin-bottom: 2px;
18 }
19
20 .peertube-select-container {
21 @include peertube-select-container(auto);
22 }
23 }
24}
25
26.redundancies-charts {
27 margin-top: 50px;
28
29 .chart-blocks {
30 display: flex;
31 justify-content: center;
32
33 .chart-block {
34 margin: 0 20px;
35 }
36 }
37}
diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts
new file mode 100644
index 000000000..4b41d1d86
--- /dev/null
+++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts
@@ -0,0 +1,178 @@
1import { Component, OnInit } from '@angular/core'
2import { Notifier, ServerService } from '@app/core'
3import { SortMeta } from 'primeng/api'
4import { ConfirmService } from '../../../core/confirm/confirm.service'
5import { RestPagination, RestTable } from '../../../shared'
6import { I18n } from '@ngx-translate/i18n-polyfill'
7import { VideoRedundanciesTarget, VideoRedundancy } from '@shared/models'
8import { peertubeLocalStorage } from '@app/shared/misc/peertube-web-storage'
9import { VideosRedundancyStats } from '@shared/models/server'
10import { BytesPipe } from 'ngx-pipes'
11import { RedundancyService } from '@app/shared/video/redundancy.service'
12
13@Component({
14 selector: 'my-video-redundancies-list',
15 templateUrl: './video-redundancies-list.component.html',
16 styleUrls: [ './video-redundancies-list.component.scss' ]
17})
18export class VideoRedundanciesListComponent extends RestTable implements OnInit {
19 private static LOCAL_STORAGE_DISPLAY_TYPE = 'video-redundancies-list-display-type'
20
21 videoRedundancies: VideoRedundancy[] = []
22 totalRecords = 0
23 rowsPerPage = 10
24
25 sort: SortMeta = { field: 'name', order: 1 }
26 pagination: RestPagination = { count: this.rowsPerPage, start: 0 }
27 displayType: VideoRedundanciesTarget = 'my-videos'
28
29 redundanciesGraphsData: { stats: VideosRedundancyStats, graphData: object, options: object }[] = []
30
31 noRedundancies = false
32
33 private bytesPipe: BytesPipe
34
35 constructor (
36 private notifier: Notifier,
37 private confirmService: ConfirmService,
38 private redundancyService: RedundancyService,
39 private serverService: ServerService,
40 private i18n: I18n
41 ) {
42 super()
43
44 this.bytesPipe = new BytesPipe()
45 }
46
47 ngOnInit () {
48 this.loadSelectLocalStorage()
49
50 this.initialize()
51
52 this.serverService.getServerStats()
53 .subscribe(res => {
54 const redundancies = res.videosRedundancy
55
56 if (redundancies.length === 0) this.noRedundancies = true
57
58 for (const r of redundancies) {
59 this.buildPieData(r)
60 }
61 })
62 }
63
64 isDisplayingRemoteVideos () {
65 return this.displayType === 'remote-videos'
66 }
67
68 getTotalSize (redundancy: VideoRedundancy) {
69 return redundancy.redundancies.files.reduce((a, b) => a + b.size, 0) +
70 redundancy.redundancies.streamingPlaylists.reduce((a, b) => a + b.size, 0)
71 }
72
73 onDisplayTypeChanged () {
74 this.pagination.start = 0
75 this.saveSelectLocalStorage()
76
77 this.loadData()
78 }
79
80 getRedundancyStrategy (redundancy: VideoRedundancy) {
81 if (redundancy.redundancies.files.length !== 0) return redundancy.redundancies.files[0].strategy
82 if (redundancy.redundancies.streamingPlaylists.length !== 0) return redundancy.redundancies.streamingPlaylists[0].strategy
83
84 return ''
85 }
86
87 buildPieData (stats: VideosRedundancyStats) {
88 const totalSize = stats.totalSize
89 ? stats.totalSize - stats.totalUsed
90 : stats.totalUsed
91
92 if (totalSize === 0) return
93
94 this.redundanciesGraphsData.push({
95 stats,
96 graphData: {
97 labels: [ this.i18n('Used'), this.i18n('Available') ],
98 datasets: [
99 {
100 data: [ stats.totalUsed, totalSize ],
101 backgroundColor: [
102 '#FF6384',
103 '#36A2EB'
104 ],
105 hoverBackgroundColor: [
106 '#FF6384',
107 '#36A2EB'
108 ]
109 }
110 ]
111 },
112 options: {
113 title: {
114 display: true,
115 text: stats.strategy
116 },
117
118 tooltips: {
119 callbacks: {
120 label: (tooltipItem: any, data: any) => {
121 const dataset = data.datasets[tooltipItem.datasetIndex]
122 let label = data.labels[tooltipItem.index]
123 if (label) label += ': '
124 else label = ''
125
126 label += this.bytesPipe.transform(dataset.data[tooltipItem.index], 1)
127 return label
128 }
129 }
130 }
131 }
132 })
133 }
134
135 async removeRedundancy (redundancy: VideoRedundancy) {
136 const message = this.i18n('Do you really want to remove this video redundancy?')
137 const res = await this.confirmService.confirm(message, this.i18n('Remove redundancy'))
138 if (res === false) return
139
140 this.redundancyService.removeVideoRedundancies(redundancy)
141 .subscribe(
142 () => {
143 this.notifier.success(this.i18n('Video redundancies removed!'))
144 this.loadData()
145 },
146
147 err => this.notifier.error(err.message)
148 )
149
150 }
151
152 protected loadData () {
153 const options = {
154 pagination: this.pagination,
155 sort: this.sort,
156 target: this.displayType
157 }
158
159 this.redundancyService.listVideoRedundancies(options)
160 .subscribe(
161 resultList => {
162 this.videoRedundancies = resultList.data
163 this.totalRecords = resultList.total
164 },
165
166 err => this.notifier.error(err.message)
167 )
168 }
169
170 private loadSelectLocalStorage () {
171 const displayType = peertubeLocalStorage.getItem(VideoRedundanciesListComponent.LOCAL_STORAGE_DISPLAY_TYPE)
172 if (displayType) this.displayType = displayType as VideoRedundanciesTarget
173 }
174
175 private saveSelectLocalStorage () {
176 peertubeLocalStorage.setItem(VideoRedundanciesListComponent.LOCAL_STORAGE_DISPLAY_TYPE, this.displayType)
177 }
178}
diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.html b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.html
new file mode 100644
index 000000000..a379520e3
--- /dev/null
+++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.html
@@ -0,0 +1,24 @@
1<div>
2 <span class="label">Url</span>
3 <a target="_blank" rel="noopener noreferrer" [href]="redundancyElement.fileUrl">{{ redundancyElement.fileUrl }}</a>
4</div>
5
6<div>
7 <span class="label">Created on</span>
8 <span>{{ redundancyElement.createdAt | date: 'medium' }}</span>
9</div>
10
11<div>
12 <span class="label">Expires on</span>
13 <span>{{ redundancyElement.expiresOn | date: 'medium' }}</span>
14</div>
15
16<div>
17 <span class="label">Size</span>
18 <span>{{ redundancyElement.size | bytes: 1 }}</span>
19</div>
20
21<div *ngIf="redundancyElement.strategy">
22 <span class="label">Strategy</span>
23 <span>{{ redundancyElement.strategy }}</span>
24</div>
diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.scss b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.scss
new file mode 100644
index 000000000..6b09fbb01
--- /dev/null
+++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.scss
@@ -0,0 +1,8 @@
1@import '_variables';
2@import '_mixins';
3
4.label {
5 display: inline-block;
6 min-width: 100px;
7 font-weight: $font-semibold;
8}
diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts
new file mode 100644
index 000000000..6f3090c08
--- /dev/null
+++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts
@@ -0,0 +1,11 @@
1import { Component, Input } from '@angular/core'
2import { FileRedundancyInformation, StreamingPlaylistRedundancyInformation } from '@shared/models'
3
4@Component({
5 selector: 'my-video-redundancy-information',
6 templateUrl: './video-redundancy-information.component.html',
7 styleUrls: [ './video-redundancy-information.component.scss' ]
8})
9export class VideoRedundancyInformationComponent {
10 @Input() redundancyElement: FileRedundancyInformation | StreamingPlaylistRedundancyInformation
11}
diff --git a/client/src/app/+admin/system/jobs/jobs.component.ts b/client/src/app/+admin/system/jobs/jobs.component.ts
index 20c8ea71a..bc40452cf 100644
--- a/client/src/app/+admin/system/jobs/jobs.component.ts
+++ b/client/src/app/+admin/system/jobs/jobs.component.ts
@@ -16,8 +16,8 @@ import { JobTypeClient } from '../../../../types/job-type-client.type'
16 styleUrls: [ './jobs.component.scss' ] 16 styleUrls: [ './jobs.component.scss' ]
17}) 17})
18export class JobsComponent extends RestTable implements OnInit { 18export class JobsComponent extends RestTable implements OnInit {
19 private static JOB_STATE_LOCAL_STORAGE_STATE = 'jobs-list-state' 19 private static LOCAL_STORAGE_STATE = 'jobs-list-state'
20 private static JOB_STATE_LOCAL_STORAGE_TYPE = 'jobs-list-type' 20 private static LOCAL_STORAGE_TYPE = 'jobs-list-type'
21 21
22 jobState: JobStateClient = 'waiting' 22 jobState: JobStateClient = 'waiting'
23 jobStates: JobStateClient[] = [ 'active', 'completed', 'failed', 'waiting', 'delayed' ] 23 jobStates: JobStateClient[] = [ 'active', 'completed', 'failed', 'waiting', 'delayed' ]
@@ -34,7 +34,8 @@ export class JobsComponent extends RestTable implements OnInit {
34 'video-file-import', 34 'video-file-import',
35 'video-import', 35 'video-import',
36 'videos-views', 36 'videos-views',
37 'activitypub-refresher' 37 'activitypub-refresher',
38 'video-redundancy'
38 ] 39 ]
39 40
40 jobs: Job[] = [] 41 jobs: Job[] = []
@@ -77,15 +78,15 @@ export class JobsComponent extends RestTable implements OnInit {
77 } 78 }
78 79
79 private loadJobStateAndType () { 80 private loadJobStateAndType () {
80 const state = peertubeLocalStorage.getItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_STATE) 81 const state = peertubeLocalStorage.getItem(JobsComponent.LOCAL_STORAGE_STATE)
81 if (state) this.jobState = state as JobState 82 if (state) this.jobState = state as JobState
82 83
83 const type = peertubeLocalStorage.getItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_TYPE) 84 const type = peertubeLocalStorage.getItem(JobsComponent.LOCAL_STORAGE_TYPE)
84 if (type) this.jobType = type as JobType 85 if (type) this.jobType = type as JobType
85 } 86 }
86 87
87 private saveJobStateAndType () { 88 private saveJobStateAndType () {
88 peertubeLocalStorage.setItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_STATE, this.jobState) 89 peertubeLocalStorage.setItem(JobsComponent.LOCAL_STORAGE_STATE, this.jobState)
89 peertubeLocalStorage.setItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_TYPE, this.jobType) 90 peertubeLocalStorage.setItem(JobsComponent.LOCAL_STORAGE_TYPE, this.jobType)
90 } 91 }
91} 92}
diff --git a/client/src/app/core/server/server.service.ts b/client/src/app/core/server/server.service.ts
index cdcbcb528..1f6cfb596 100644
--- a/client/src/app/core/server/server.service.ts
+++ b/client/src/app/core/server/server.service.ts
@@ -9,6 +9,7 @@ import { VideoConstant } from '../../../../../shared/models/videos'
9import { isDefaultLocale, peertubeTranslate } from '../../../../../shared/models/i18n' 9import { isDefaultLocale, peertubeTranslate } from '../../../../../shared/models/i18n'
10import { getDevLocale, isOnDevLocale } from '@app/shared/i18n/i18n-utils' 10import { getDevLocale, isOnDevLocale } from '@app/shared/i18n/i18n-utils'
11import { sortBy } from '@app/shared/misc/utils' 11import { sortBy } from '@app/shared/misc/utils'
12import { ServerStats } from '@shared/models/server'
12 13
13@Injectable() 14@Injectable()
14export class ServerService { 15export class ServerService {
@@ -16,6 +17,8 @@ export class ServerService {
16 private static BASE_VIDEO_URL = environment.apiUrl + '/api/v1/videos/' 17 private static BASE_VIDEO_URL = environment.apiUrl + '/api/v1/videos/'
17 private static BASE_VIDEO_PLAYLIST_URL = environment.apiUrl + '/api/v1/video-playlists/' 18 private static BASE_VIDEO_PLAYLIST_URL = environment.apiUrl + '/api/v1/video-playlists/'
18 private static BASE_LOCALE_URL = environment.apiUrl + '/client/locales/' 19 private static BASE_LOCALE_URL = environment.apiUrl + '/client/locales/'
20 private static BASE_STATS_URL = environment.apiUrl + '/api/v1/server/stats'
21
19 private static CONFIG_LOCAL_STORAGE_KEY = 'server-config' 22 private static CONFIG_LOCAL_STORAGE_KEY = 'server-config'
20 23
21 configReloaded = new Subject<void>() 24 configReloaded = new Subject<void>()
@@ -235,6 +238,10 @@ export class ServerService {
235 return this.localeObservable.pipe(first()) 238 return this.localeObservable.pipe(first())
236 } 239 }
237 240
241 getServerStats () {
242 return this.http.get<ServerStats>(ServerService.BASE_STATS_URL)
243 }
244
238 private loadAttributeEnum <T extends string | number> ( 245 private loadAttributeEnum <T extends string | number> (
239 baseUrl: string, 246 baseUrl: string,
240 attributeName: 'categories' | 'licences' | 'languages' | 'privacies', 247 attributeName: 'categories' | 'licences' | 'languages' | 'privacies',
diff --git a/client/src/app/shared/images/global-icon.component.ts b/client/src/app/shared/images/global-icon.component.ts
index 806aca347..b6e641228 100644
--- a/client/src/app/shared/images/global-icon.component.ts
+++ b/client/src/app/shared/images/global-icon.component.ts
@@ -1,6 +1,5 @@
1import { ChangeDetectionStrategy, Component, ElementRef, Input, OnInit } from '@angular/core' 1import { ChangeDetectionStrategy, Component, ElementRef, Input, OnInit } from '@angular/core'
2import { HooksService } from '@app/core/plugins/hooks.service' 2import { HooksService } from '@app/core/plugins/hooks.service'
3import { I18n } from '@ngx-translate/i18n-polyfill'
4 3
5const icons = { 4const icons = {
6 'add': require('!!raw-loader?!../../../assets/images/global/add.svg'), 5 'add': require('!!raw-loader?!../../../assets/images/global/add.svg'),
diff --git a/client/src/app/shared/instance/instance-statistics.component.ts b/client/src/app/shared/instance/instance-statistics.component.ts
index 8ec728f05..40aa8a4c0 100644
--- a/client/src/app/shared/instance/instance-statistics.component.ts
+++ b/client/src/app/shared/instance/instance-statistics.component.ts
@@ -1,9 +1,6 @@
1import { map } from 'rxjs/operators'
2import { HttpClient } from '@angular/common/http'
3import { Component, OnInit } from '@angular/core' 1import { Component, OnInit } from '@angular/core'
4import { I18n } from '@ngx-translate/i18n-polyfill'
5import { ServerStats } from '@shared/models/server' 2import { ServerStats } from '@shared/models/server'
6import { environment } from '../../../environments/environment' 3import { ServerService } from '@app/core'
7 4
8@Component({ 5@Component({
9 selector: 'my-instance-statistics', 6 selector: 'my-instance-statistics',
@@ -11,27 +8,15 @@ import { environment } from '../../../environments/environment'
11 styleUrls: [ './instance-statistics.component.scss' ] 8 styleUrls: [ './instance-statistics.component.scss' ]
12}) 9})
13export class InstanceStatisticsComponent implements OnInit { 10export class InstanceStatisticsComponent implements OnInit {
14 private static BASE_STATS_URL = environment.apiUrl + '/api/v1/server/stats'
15
16 serverStats: ServerStats = null 11 serverStats: ServerStats = null
17 12
18 constructor ( 13 constructor (
19 private http: HttpClient, 14 private serverService: ServerService
20 private i18n: I18n
21 ) { 15 ) {
22 } 16 }
23 17
24 ngOnInit () { 18 ngOnInit () {
25 this.getStats() 19 this.serverService.getServerStats()
26 .subscribe( 20 .subscribe(res => this.serverStats = res)
27 res => {
28 this.serverStats = res
29 }
30 )
31 }
32
33 getStats () {
34 return this.http
35 .get<ServerStats>(InstanceStatisticsComponent.BASE_STATS_URL)
36 } 21 }
37} 22}
diff --git a/client/src/app/shared/shared.module.ts b/client/src/app/shared/shared.module.ts
index b2eb13f73..d06d37d8c 100644
--- a/client/src/app/shared/shared.module.ts
+++ b/client/src/app/shared/shared.module.ts
@@ -98,6 +98,7 @@ import { FollowService } from '@app/shared/instance/follow.service'
98import { MultiSelectModule } from 'primeng/multiselect' 98import { MultiSelectModule } from 'primeng/multiselect'
99import { FeatureBooleanComponent } from '@app/shared/instance/feature-boolean.component' 99import { FeatureBooleanComponent } from '@app/shared/instance/feature-boolean.component'
100import { InputReadonlyCopyComponent } from '@app/shared/forms/input-readonly-copy.component' 100import { InputReadonlyCopyComponent } from '@app/shared/forms/input-readonly-copy.component'
101import { RedundancyService } from '@app/shared/video/redundancy.service'
101 102
102@NgModule({ 103@NgModule({
103 imports: [ 104 imports: [
@@ -300,6 +301,7 @@ import { InputReadonlyCopyComponent } from '@app/shared/forms/input-readonly-cop
300 UserNotificationService, 301 UserNotificationService,
301 302
302 FollowService, 303 FollowService,
304 RedundancyService,
303 305
304 I18n 306 I18n
305 ] 307 ]
diff --git a/client/src/app/shared/video/redundancy.service.ts b/client/src/app/shared/video/redundancy.service.ts
new file mode 100644
index 000000000..fb918d73b
--- /dev/null
+++ b/client/src/app/shared/video/redundancy.service.ts
@@ -0,0 +1,73 @@
1import { catchError, map, toArray } from 'rxjs/operators'
2import { HttpClient, HttpParams } from '@angular/common/http'
3import { Injectable } from '@angular/core'
4import { RestExtractor, RestPagination, RestService } from '@app/shared/rest'
5import { SortMeta } from 'primeng/api'
6import { ResultList, Video, VideoRedundanciesTarget, VideoRedundancy } from '@shared/models'
7import { concat, Observable } from 'rxjs'
8import { environment } from '../../../environments/environment'
9
10@Injectable()
11export class RedundancyService {
12 static BASE_REDUNDANCY_URL = environment.apiUrl + '/api/v1/server/redundancy'
13
14 constructor (
15 private authHttp: HttpClient,
16 private restService: RestService,
17 private restExtractor: RestExtractor
18 ) { }
19
20 updateRedundancy (host: string, redundancyAllowed: boolean) {
21 const url = RedundancyService.BASE_REDUNDANCY_URL + '/' + host
22
23 const body = { redundancyAllowed }
24
25 return this.authHttp.put(url, body)
26 .pipe(
27 map(this.restExtractor.extractDataBool),
28 catchError(err => this.restExtractor.handleError(err))
29 )
30 }
31
32 listVideoRedundancies (options: {
33 pagination: RestPagination,
34 sort: SortMeta,
35 target?: VideoRedundanciesTarget
36 }): Observable<ResultList<VideoRedundancy>> {
37 const { pagination, sort, target } = options
38
39 let params = new HttpParams()
40 params = this.restService.addRestGetParams(params, pagination, sort)
41
42 if (target) params = params.append('target', target)
43
44 return this.authHttp.get<ResultList<VideoRedundancy>>(RedundancyService.BASE_REDUNDANCY_URL + '/videos', { params })
45 .pipe(
46 catchError(res => this.restExtractor.handleError(res))
47 )
48 }
49
50 addVideoRedundancy (video: Video) {
51 return this.authHttp.post(RedundancyService.BASE_REDUNDANCY_URL + '/videos', { videoId: video.id })
52 .pipe(
53 catchError(res => this.restExtractor.handleError(res))
54 )
55 }
56
57 removeVideoRedundancies (redundancy: VideoRedundancy) {
58 const observables = redundancy.redundancies.streamingPlaylists.map(r => r.id)
59 .concat(redundancy.redundancies.files.map(r => r.id))
60 .map(id => this.removeRedundancy(id))
61
62 return concat(...observables)
63 .pipe(toArray())
64 }
65
66 private removeRedundancy (redundancyId: number) {
67 return this.authHttp.delete(RedundancyService.BASE_REDUNDANCY_URL + '/videos/' + redundancyId)
68 .pipe(
69 map(this.restExtractor.extractDataBool),
70 catchError(res => this.restExtractor.handleError(res))
71 )
72 }
73}
diff --git a/client/src/app/shared/video/video-actions-dropdown.component.ts b/client/src/app/shared/video/video-actions-dropdown.component.ts
index afdeab18d..390d74c52 100644
--- a/client/src/app/shared/video/video-actions-dropdown.component.ts
+++ b/client/src/app/shared/video/video-actions-dropdown.component.ts
@@ -14,6 +14,7 @@ import { VideoBlacklistComponent } from '@app/shared/video/modals/video-blacklis
14import { VideoBlacklistService } from '@app/shared/video-blacklist' 14import { VideoBlacklistService } from '@app/shared/video-blacklist'
15import { ScreenService } from '@app/shared/misc/screen.service' 15import { ScreenService } from '@app/shared/misc/screen.service'
16import { VideoCaption } from '@shared/models' 16import { VideoCaption } from '@shared/models'
17import { RedundancyService } from '@app/shared/video/redundancy.service'
17 18
18export type VideoActionsDisplayType = { 19export type VideoActionsDisplayType = {
19 playlist?: boolean 20 playlist?: boolean
@@ -22,6 +23,7 @@ export type VideoActionsDisplayType = {
22 blacklist?: boolean 23 blacklist?: boolean
23 delete?: boolean 24 delete?: boolean
24 report?: boolean 25 report?: boolean
26 duplicate?: boolean
25} 27}
26 28
27@Component({ 29@Component({
@@ -46,7 +48,8 @@ export class VideoActionsDropdownComponent implements OnChanges {
46 update: true, 48 update: true,
47 blacklist: true, 49 blacklist: true,
48 delete: true, 50 delete: true,
49 report: true 51 report: true,
52 duplicate: true
50 } 53 }
51 @Input() placement = 'left' 54 @Input() placement = 'left'
52 55
@@ -74,6 +77,7 @@ export class VideoActionsDropdownComponent implements OnChanges {
74 private screenService: ScreenService, 77 private screenService: ScreenService,
75 private videoService: VideoService, 78 private videoService: VideoService,
76 private blocklistService: BlocklistService, 79 private blocklistService: BlocklistService,
80 private redundancyService: RedundancyService,
77 private i18n: I18n 81 private i18n: I18n
78 ) { } 82 ) { }
79 83
@@ -144,6 +148,10 @@ export class VideoActionsDropdownComponent implements OnChanges {
144 return this.video && this.video instanceof VideoDetails && this.video.downloadEnabled 148 return this.video && this.video instanceof VideoDetails && this.video.downloadEnabled
145 } 149 }
146 150
151 canVideoBeDuplicated () {
152 return this.video.canBeDuplicatedBy(this.user)
153 }
154
147 /* Action handlers */ 155 /* Action handlers */
148 156
149 async unblacklistVideo () { 157 async unblacklistVideo () {
@@ -186,6 +194,18 @@ export class VideoActionsDropdownComponent implements OnChanges {
186 ) 194 )
187 } 195 }
188 196
197 duplicateVideo () {
198 this.redundancyService.addVideoRedundancy(this.video)
199 .subscribe(
200 () => {
201 const message = this.i18n('This video will be duplicated by your instance.')
202 this.notifier.success(message)
203 },
204
205 err => this.notifier.error(err.message)
206 )
207 }
208
189 onVideoBlacklisted () { 209 onVideoBlacklisted () {
190 this.videoBlacklisted.emit() 210 this.videoBlacklisted.emit()
191 } 211 }
@@ -234,6 +254,12 @@ export class VideoActionsDropdownComponent implements OnChanges {
234 isDisplayed: () => this.authService.isLoggedIn() && this.displayOptions.blacklist && this.isVideoUnblacklistable() 254 isDisplayed: () => this.authService.isLoggedIn() && this.displayOptions.blacklist && this.isVideoUnblacklistable()
235 }, 255 },
236 { 256 {
257 label: this.i18n('Duplicate (redundancy)'),
258 handler: () => this.duplicateVideo(),
259 isDisplayed: () => this.authService.isLoggedIn() && this.displayOptions.duplicate && this.canVideoBeDuplicated(),
260 iconName: 'cloud-download'
261 },
262 {
237 label: this.i18n('Delete'), 263 label: this.i18n('Delete'),
238 handler: () => this.removeVideo(), 264 handler: () => this.removeVideo(),
239 isDisplayed: () => this.authService.isLoggedIn() && this.displayOptions.delete && this.isVideoRemovable(), 265 isDisplayed: () => this.authService.isLoggedIn() && this.displayOptions.delete && this.isVideoRemovable(),
diff --git a/client/src/app/shared/video/video-miniature.component.ts b/client/src/app/shared/video/video-miniature.component.ts
index 598a7a983..1dfb3eec7 100644
--- a/client/src/app/shared/video/video-miniature.component.ts
+++ b/client/src/app/shared/video/video-miniature.component.ts
@@ -64,7 +64,8 @@ export class VideoMiniatureComponent implements OnInit {
64 update: true, 64 update: true,
65 blacklist: true, 65 blacklist: true,
66 delete: true, 66 delete: true,
67 report: true 67 report: true,
68 duplicate: false
68 } 69 }
69 showActions = false 70 showActions = false
70 serverConfig: ServerConfig 71 serverConfig: ServerConfig
diff --git a/client/src/app/shared/video/video.model.ts b/client/src/app/shared/video/video.model.ts
index fb98d5382..9eeaf41b0 100644
--- a/client/src/app/shared/video/video.model.ts
+++ b/client/src/app/shared/video/video.model.ts
@@ -152,4 +152,8 @@ export class Video implements VideoServerModel {
152 isUpdatableBy (user: AuthUser) { 152 isUpdatableBy (user: AuthUser) {
153 return user && this.isLocal === true && (this.account.name === user.username || user.hasRight(UserRight.UPDATE_ANY_VIDEO)) 153 return user && this.isLocal === true && (this.account.name === user.username || user.hasRight(UserRight.UPDATE_ANY_VIDEO))
154 } 154 }
155
156 canBeDuplicatedBy (user: AuthUser) {
157 return user && this.isLocal === false && user.hasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES)
158 }
155} 159}
diff --git a/client/yarn.lock b/client/yarn.lock
index 0855a2570..c900ae549 100644
--- a/client/yarn.lock
+++ b/client/yarn.lock
@@ -2586,6 +2586,29 @@ chardet@^0.7.0:
2586 resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e" 2586 resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e"
2587 integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA== 2587 integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==
2588 2588
2589chart.js@^2.9.3:
2590 version "2.9.3"
2591 resolved "https://registry.yarnpkg.com/chart.js/-/chart.js-2.9.3.tgz#ae3884114dafd381bc600f5b35a189138aac1ef7"
2592 integrity sha512-+2jlOobSk52c1VU6fzkh3UwqHMdSlgH1xFv9FKMqHiNCpXsGPQa/+81AFa+i3jZ253Mq9aAycPwDjnn1XbRNNw==
2593 dependencies:
2594 chartjs-color "^2.1.0"
2595 moment "^2.10.2"
2596
2597chartjs-color-string@^0.6.0:
2598 version "0.6.0"
2599 resolved "https://registry.yarnpkg.com/chartjs-color-string/-/chartjs-color-string-0.6.0.tgz#1df096621c0e70720a64f4135ea171d051402f71"
2600 integrity sha512-TIB5OKn1hPJvO7JcteW4WY/63v6KwEdt6udfnDE9iCAZgy+V4SrbSxoIbTw/xkUIapjEI4ExGtD0+6D3KyFd7A==
2601 dependencies:
2602 color-name "^1.0.0"
2603
2604chartjs-color@^2.1.0:
2605 version "2.4.1"
2606 resolved "https://registry.yarnpkg.com/chartjs-color/-/chartjs-color-2.4.1.tgz#6118bba202fe1ea79dd7f7c0f9da93467296c3b0"
2607 integrity sha512-haqOg1+Yebys/Ts/9bLo/BqUcONQOdr/hoEr2LLTRl6C5LXctUdHxsCYfvQVg5JIxITrfCNUDr4ntqmQk9+/0w==
2608 dependencies:
2609 chartjs-color-string "^0.6.0"
2610 color-convert "^1.9.3"
2611
2589check-types@^8.0.3: 2612check-types@^8.0.3:
2590 version "8.0.3" 2613 version "8.0.3"
2591 resolved "https://registry.yarnpkg.com/check-types/-/check-types-8.0.3.tgz#3356cca19c889544f2d7a95ed49ce508a0ecf552" 2614 resolved "https://registry.yarnpkg.com/check-types/-/check-types-8.0.3.tgz#3356cca19c889544f2d7a95ed49ce508a0ecf552"
@@ -2800,7 +2823,7 @@ collection-visit@^1.0.0:
2800 map-visit "^1.0.0" 2823 map-visit "^1.0.0"
2801 object-visit "^1.0.0" 2824 object-visit "^1.0.0"
2802 2825
2803color-convert@^1.9.0: 2826color-convert@^1.9.0, color-convert@^1.9.3:
2804 version "1.9.3" 2827 version "1.9.3"
2805 resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" 2828 resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8"
2806 integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== 2829 integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==
@@ -2812,6 +2835,11 @@ color-name@1.1.3:
2812 resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" 2835 resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25"
2813 integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= 2836 integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=
2814 2837
2838color-name@^1.0.0:
2839 version "1.1.4"
2840 resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2"
2841 integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==
2842
2815colors@1.1.2: 2843colors@1.1.2:
2816 version "1.1.2" 2844 version "1.1.2"
2817 resolved "https://registry.yarnpkg.com/colors/-/colors-1.1.2.tgz#168a4701756b6a7f51a12ce0c97bfa28c084ed63" 2845 resolved "https://registry.yarnpkg.com/colors/-/colors-1.1.2.tgz#168a4701756b6a7f51a12ce0c97bfa28c084ed63"
@@ -6941,6 +6969,11 @@ mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1:
6941 dependencies: 6969 dependencies:
6942 minimist "0.0.8" 6970 minimist "0.0.8"
6943 6971
6972moment@^2.10.2:
6973 version "2.24.0"
6974 resolved "https://registry.yarnpkg.com/moment/-/moment-2.24.0.tgz#0d055d53f5052aa653c9f6eb68bb5d12bf5c2b5b"
6975 integrity sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==
6976
6944mousetrap@^1.6.0: 6977mousetrap@^1.6.0:
6945 version "1.6.3" 6978 version "1.6.3"
6946 resolved "https://registry.yarnpkg.com/mousetrap/-/mousetrap-1.6.3.tgz#80fee49665fd478bccf072c9d46bdf1bfed3558a" 6979 resolved "https://registry.yarnpkg.com/mousetrap/-/mousetrap-1.6.3.tgz#80fee49665fd478bccf072c9d46bdf1bfed3558a"
diff --git a/config/test.yaml b/config/test.yaml
index 5758c1887..74979f3a7 100644
--- a/config/test.yaml
+++ b/config/test.yaml
@@ -40,18 +40,18 @@ contact_form:
40 40
41redundancy: 41redundancy:
42 videos: 42 videos:
43 check_interval: '10 minutes' 43 check_interval: '1 minute'
44 strategies: 44 strategies:
45 - 45 -
46 size: '10MB' 46 size: '1000MB'
47 min_lifetime: '10 minutes' 47 min_lifetime: '10 minutes'
48 strategy: 'most-views' 48 strategy: 'most-views'
49 - 49 -
50 size: '10MB' 50 size: '1000MB'
51 min_lifetime: '10 minutes' 51 min_lifetime: '10 minutes'
52 strategy: 'trending' 52 strategy: 'trending'
53 - 53 -
54 size: '10MB' 54 size: '1000MB'
55 min_lifetime: '10 minutes' 55 min_lifetime: '10 minutes'
56 strategy: 'recently-added' 56 strategy: 'recently-added'
57 min_views: 1 57 min_views: 1
diff --git a/package.json b/package.json
index d4858725a..fed390c82 100644
--- a/package.json
+++ b/package.json
@@ -41,7 +41,7 @@
41 "i18n:create-custom-files": "node ./dist/scripts/i18n/create-custom-files.js", 41 "i18n:create-custom-files": "node ./dist/scripts/i18n/create-custom-files.js",
42 "reset-password": "node ./dist/scripts/reset-password.js", 42 "reset-password": "node ./dist/scripts/reset-password.js",
43 "play": "scripty", 43 "play": "scripty",
44 "dev": "scripty", 44 "dev": "sh ./scripts/dev/index.sh",
45 "dev:server": "sh ./scripts/dev/server.sh", 45 "dev:server": "sh ./scripts/dev/server.sh",
46 "dev:embed": "scripty", 46 "dev:embed": "scripty",
47 "dev:client": "sh ./scripts/dev/client.sh", 47 "dev:client": "sh ./scripts/dev/client.sh",
diff --git a/scripts/dev/index.sh b/scripts/dev/index.sh
index d221d2fc8..9e89516b8 100755
--- a/scripts/dev/index.sh
+++ b/scripts/dev/index.sh
@@ -3,5 +3,5 @@
3set -eu 3set -eu
4 4
5NODE_ENV=test npm run concurrently -- -k \ 5NODE_ENV=test npm run concurrently -- -k \
6 "npm run dev:client -- --skip-server" \ 6 "sh scripts/dev/client.sh --skip-server" \
7 "npm run dev:server" 7 "sh scripts/dev/server.sh"
diff --git a/server/controllers/api/server/follows.ts b/server/controllers/api/server/follows.ts
index 29a403a43..c69de21fb 100644
--- a/server/controllers/api/server/follows.ts
+++ b/server/controllers/api/server/follows.ts
@@ -24,7 +24,7 @@ import {
24} from '../../../middlewares/validators' 24} from '../../../middlewares/validators'
25import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 25import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
26import { JobQueue } from '../../../lib/job-queue' 26import { JobQueue } from '../../../lib/job-queue'
27import { removeRedundancyOf } from '../../../lib/redundancy' 27import { removeRedundanciesOfServer } from '../../../lib/redundancy'
28import { sequelizeTypescript } from '../../../initializers/database' 28import { sequelizeTypescript } from '../../../initializers/database'
29import { autoFollowBackIfNeeded } from '../../../lib/activitypub/follow' 29import { autoFollowBackIfNeeded } from '../../../lib/activitypub/follow'
30 30
@@ -153,7 +153,7 @@ async function removeFollowing (req: express.Request, res: express.Response) {
153 await server.save({ transaction: t }) 153 await server.save({ transaction: t })
154 154
155 // Async, could be long 155 // Async, could be long
156 removeRedundancyOf(server.id) 156 removeRedundanciesOfServer(server.id)
157 .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, err)) 157 .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, err))
158 158
159 await follow.destroy({ transaction: t }) 159 await follow.destroy({ transaction: t })
diff --git a/server/controllers/api/server/redundancy.ts b/server/controllers/api/server/redundancy.ts
index 4ea6164a3..a11c1a74f 100644
--- a/server/controllers/api/server/redundancy.ts
+++ b/server/controllers/api/server/redundancy.ts
@@ -1,9 +1,24 @@
1import * as express from 'express' 1import * as express from 'express'
2import { UserRight } from '../../../../shared/models/users' 2import { UserRight } from '../../../../shared/models/users'
3import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares' 3import {
4import { updateServerRedundancyValidator } from '../../../middlewares/validators/redundancy' 4 asyncMiddleware,
5import { removeRedundancyOf } from '../../../lib/redundancy' 5 authenticate,
6 ensureUserHasRight,
7 paginationValidator,
8 setDefaultPagination,
9 setDefaultVideoRedundanciesSort,
10 videoRedundanciesSortValidator
11} from '../../../middlewares'
12import {
13 listVideoRedundanciesValidator,
14 updateServerRedundancyValidator,
15 addVideoRedundancyValidator,
16 removeVideoRedundancyValidator
17} from '../../../middlewares/validators/redundancy'
18import { removeRedundanciesOfServer, removeVideoRedundancy } from '../../../lib/redundancy'
6import { logger } from '../../../helpers/logger' 19import { logger } from '../../../helpers/logger'
20import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy'
21import { JobQueue } from '@server/lib/job-queue'
7 22
8const serverRedundancyRouter = express.Router() 23const serverRedundancyRouter = express.Router()
9 24
@@ -14,6 +29,31 @@ serverRedundancyRouter.put('/redundancy/:host',
14 asyncMiddleware(updateRedundancy) 29 asyncMiddleware(updateRedundancy)
15) 30)
16 31
32serverRedundancyRouter.get('/redundancy/videos',
33 authenticate,
34 ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES),
35 listVideoRedundanciesValidator,
36 paginationValidator,
37 videoRedundanciesSortValidator,
38 setDefaultVideoRedundanciesSort,
39 setDefaultPagination,
40 asyncMiddleware(listVideoRedundancies)
41)
42
43serverRedundancyRouter.post('/redundancy/videos',
44 authenticate,
45 ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES),
46 addVideoRedundancyValidator,
47 asyncMiddleware(addVideoRedundancy)
48)
49
50serverRedundancyRouter.delete('/redundancy/videos/:redundancyId',
51 authenticate,
52 ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES),
53 removeVideoRedundancyValidator,
54 asyncMiddleware(removeVideoRedundancyController)
55)
56
17// --------------------------------------------------------------------------- 57// ---------------------------------------------------------------------------
18 58
19export { 59export {
@@ -22,6 +62,42 @@ export {
22 62
23// --------------------------------------------------------------------------- 63// ---------------------------------------------------------------------------
24 64
65async function listVideoRedundancies (req: express.Request, res: express.Response) {
66 const resultList = await VideoRedundancyModel.listForApi({
67 start: req.query.start,
68 count: req.query.count,
69 sort: req.query.sort,
70 target: req.query.target,
71 strategy: req.query.strategy
72 })
73
74 const result = {
75 total: resultList.total,
76 data: resultList.data.map(r => VideoRedundancyModel.toFormattedJSONStatic(r))
77 }
78
79 return res.json(result)
80}
81
82async function addVideoRedundancy (req: express.Request, res: express.Response) {
83 const payload = {
84 videoId: res.locals.onlyVideo.id
85 }
86
87 await JobQueue.Instance.createJob({
88 type: 'video-redundancy',
89 payload
90 })
91
92 return res.sendStatus(204)
93}
94
95async function removeVideoRedundancyController (req: express.Request, res: express.Response) {
96 await removeVideoRedundancy(res.locals.videoRedundancy)
97
98 return res.sendStatus(204)
99}
100
25async function updateRedundancy (req: express.Request, res: express.Response) { 101async function updateRedundancy (req: express.Request, res: express.Response) {
26 const server = res.locals.server 102 const server = res.locals.server
27 103
@@ -30,7 +106,7 @@ async function updateRedundancy (req: express.Request, res: express.Response) {
30 await server.save() 106 await server.save()
31 107
32 // Async, could be long 108 // Async, could be long
33 removeRedundancyOf(server.id) 109 removeRedundanciesOfServer(server.id)
34 .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, { err })) 110 .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, { err }))
35 111
36 return res.sendStatus(204) 112 return res.sendStatus(204)
diff --git a/server/controllers/api/server/stats.ts b/server/controllers/api/server/stats.ts
index 3616c074d..6d508a481 100644
--- a/server/controllers/api/server/stats.ts
+++ b/server/controllers/api/server/stats.ts
@@ -10,6 +10,7 @@ import { ROUTE_CACHE_LIFETIME } from '../../../initializers/constants'
10import { cacheRoute } from '../../../middlewares/cache' 10import { cacheRoute } from '../../../middlewares/cache'
11import { VideoFileModel } from '../../../models/video/video-file' 11import { VideoFileModel } from '../../../models/video/video-file'
12import { CONFIG } from '../../../initializers/config' 12import { CONFIG } from '../../../initializers/config'
13import { VideoRedundancyStrategyWithManual } from '@shared/models'
13 14
14const statsRouter = express.Router() 15const statsRouter = express.Router()
15 16
@@ -25,8 +26,15 @@ async function getStats (req: express.Request, res: express.Response) {
25 const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() 26 const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats()
26 const { totalLocalVideoFilesSize } = await VideoFileModel.getStats() 27 const { totalLocalVideoFilesSize } = await VideoFileModel.getStats()
27 28
29 const strategies: { strategy: VideoRedundancyStrategyWithManual, size: number }[] = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES
30 .map(r => ({
31 strategy: r.strategy,
32 size: r.size
33 }))
34 strategies.push({ strategy: 'manual', size: null })
35
28 const videosRedundancyStats = await Promise.all( 36 const videosRedundancyStats = await Promise.all(
29 CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.map(r => { 37 strategies.map(r => {
30 return VideoRedundancyModel.getStats(r.strategy) 38 return VideoRedundancyModel.getStats(r.strategy)
31 .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size })) 39 .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size }))
32 }) 40 })
diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts
index 21d5c53ca..c5b3b4d9f 100644
--- a/server/helpers/custom-validators/activitypub/cache-file.ts
+++ b/server/helpers/custom-validators/activitypub/cache-file.ts
@@ -6,7 +6,7 @@ import { CacheFileObject } from '../../../../shared/models/activitypub/objects'
6function isCacheFileObjectValid (object: CacheFileObject) { 6function isCacheFileObjectValid (object: CacheFileObject) {
7 return exists(object) && 7 return exists(object) &&
8 object.type === 'CacheFile' && 8 object.type === 'CacheFile' &&
9 isDateValid(object.expires) && 9 (object.expires === null || isDateValid(object.expires)) &&
10 isActivityPubUrlValid(object.object) && 10 isActivityPubUrlValid(object.object) &&
11 (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url)) 11 (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url))
12} 12}
diff --git a/server/helpers/custom-validators/video-redundancies.ts b/server/helpers/custom-validators/video-redundancies.ts
new file mode 100644
index 000000000..50a559c4f
--- /dev/null
+++ b/server/helpers/custom-validators/video-redundancies.ts
@@ -0,0 +1,12 @@
1import { exists } from './misc'
2
3function isVideoRedundancyTarget (value: any) {
4 return exists(value) &&
5 (value === 'my-videos' || value === 'remote-videos')
6}
7
8// ---------------------------------------------------------------------------
9
10export {
11 isVideoRedundancyTarget
12}
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index 3a99518c6..8a5d030df 100644
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -9,12 +9,12 @@ import { promisify2 } from './core-utils'
9import { MVideo } from '@server/typings/models/video/video' 9import { MVideo } from '@server/typings/models/video/video'
10import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/typings/models/video/video-file' 10import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/typings/models/video/video-file'
11import { isStreamingPlaylist, MStreamingPlaylistVideo } from '@server/typings/models/video/video-streaming-playlist' 11import { isStreamingPlaylist, MStreamingPlaylistVideo } from '@server/typings/models/video/video-streaming-playlist'
12import { STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' 12import { WEBSERVER } from '@server/initializers/constants'
13import * as parseTorrent from 'parse-torrent' 13import * as parseTorrent from 'parse-torrent'
14import * as magnetUtil from 'magnet-uri' 14import * as magnetUtil from 'magnet-uri'
15import { isArray } from '@server/helpers/custom-validators/misc' 15import { isArray } from '@server/helpers/custom-validators/misc'
16import { extractVideo } from '@server/lib/videos' 16import { extractVideo } from '@server/lib/videos'
17import { getTorrentFileName, getVideoFilename, getVideoFilePath } from '@server/lib/video-paths' 17import { getTorrentFileName, getVideoFilePath } from '@server/lib/video-paths'
18 18
19const createTorrentPromise = promisify2<string, any, any>(createTorrent) 19const createTorrentPromise = promisify2<string, any, any>(createTorrent)
20 20
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 7fd77f3e8..fd8bf09fc 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -1,6 +1,6 @@
1import { IConfig } from 'config' 1import { IConfig } from 'config'
2import { dirname, join } from 'path' 2import { dirname, join } from 'path'
3import { VideosRedundancy } from '../../shared/models' 3import { VideosRedundancyStrategy } from '../../shared/models'
4// Do not use barrels, remain constants as independent as possible 4// Do not use barrels, remain constants as independent as possible
5import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils' 5import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils'
6import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' 6import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
@@ -304,7 +304,7 @@ function getLocalConfigFilePath () {
304 return join(dirname(configSources[ 0 ].name), filename + '.json') 304 return join(dirname(configSources[ 0 ].name), filename + '.json')
305} 305}
306 306
307function buildVideosRedundancy (objs: any[]): VideosRedundancy[] { 307function buildVideosRedundancy (objs: any[]): VideosRedundancyStrategy[] {
308 if (!objs) return [] 308 if (!objs) return []
309 309
310 if (!Array.isArray(objs)) return objs 310 if (!Array.isArray(objs)) return objs
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 032f63c8f..e01ab8943 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -14,7 +14,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
14 14
15// --------------------------------------------------------------------------- 15// ---------------------------------------------------------------------------
16 16
17const LAST_MIGRATION_VERSION = 470 17const LAST_MIGRATION_VERSION = 475
18 18
19// --------------------------------------------------------------------------- 19// ---------------------------------------------------------------------------
20 20
@@ -73,7 +73,9 @@ const SORTABLE_COLUMNS = {
73 73
74 PLUGINS: [ 'name', 'createdAt', 'updatedAt' ], 74 PLUGINS: [ 'name', 'createdAt', 'updatedAt' ],
75 75
76 AVAILABLE_PLUGINS: [ 'npmName', 'popularity' ] 76 AVAILABLE_PLUGINS: [ 'npmName', 'popularity' ],
77
78 VIDEO_REDUNDANCIES: [ 'name' ]
77} 79}
78 80
79const OAUTH_LIFETIME = { 81const OAUTH_LIFETIME = {
@@ -117,45 +119,44 @@ const REMOTE_SCHEME = {
117 WS: 'wss' 119 WS: 'wss'
118} 120}
119 121
120// TODO: remove 'video-file' 122const JOB_ATTEMPTS: { [id in JobType]: number } = {
121const JOB_ATTEMPTS: { [id in (JobType | 'video-file')]: number } = {
122 'activitypub-http-broadcast': 5, 123 'activitypub-http-broadcast': 5,
123 'activitypub-http-unicast': 5, 124 'activitypub-http-unicast': 5,
124 'activitypub-http-fetcher': 5, 125 'activitypub-http-fetcher': 5,
125 'activitypub-follow': 5, 126 'activitypub-follow': 5,
126 'video-file-import': 1, 127 'video-file-import': 1,
127 'video-transcoding': 1, 128 'video-transcoding': 1,
128 'video-file': 1,
129 'video-import': 1, 129 'video-import': 1,
130 'email': 5, 130 'email': 5,
131 'videos-views': 1, 131 'videos-views': 1,
132 'activitypub-refresher': 1 132 'activitypub-refresher': 1,
133 'video-redundancy': 1
133} 134}
134const JOB_CONCURRENCY: { [id in (JobType | 'video-file')]: number } = { 135const JOB_CONCURRENCY: { [id in JobType]: number } = {
135 'activitypub-http-broadcast': 1, 136 'activitypub-http-broadcast': 1,
136 'activitypub-http-unicast': 5, 137 'activitypub-http-unicast': 5,
137 'activitypub-http-fetcher': 1, 138 'activitypub-http-fetcher': 1,
138 'activitypub-follow': 1, 139 'activitypub-follow': 1,
139 'video-file-import': 1, 140 'video-file-import': 1,
140 'video-transcoding': 1, 141 'video-transcoding': 1,
141 'video-file': 1,
142 'video-import': 1, 142 'video-import': 1,
143 'email': 5, 143 'email': 5,
144 'videos-views': 1, 144 'videos-views': 1,
145 'activitypub-refresher': 1 145 'activitypub-refresher': 1,
146 'video-redundancy': 1
146} 147}
147const JOB_TTL: { [id in (JobType | 'video-file')]: number } = { 148const JOB_TTL: { [id in JobType]: number } = {
148 'activitypub-http-broadcast': 60000 * 10, // 10 minutes 149 'activitypub-http-broadcast': 60000 * 10, // 10 minutes
149 'activitypub-http-unicast': 60000 * 10, // 10 minutes 150 'activitypub-http-unicast': 60000 * 10, // 10 minutes
150 'activitypub-http-fetcher': 60000 * 10, // 10 minutes 151 'activitypub-http-fetcher': 60000 * 10, // 10 minutes
151 'activitypub-follow': 60000 * 10, // 10 minutes 152 'activitypub-follow': 60000 * 10, // 10 minutes
152 'video-file-import': 1000 * 3600, // 1 hour 153 'video-file-import': 1000 * 3600, // 1 hour
153 'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long 154 'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long
154 'video-file': 1000 * 3600 * 48, // 2 days, transcoding could be long
155 'video-import': 1000 * 3600 * 2, // hours 155 'video-import': 1000 * 3600 * 2, // hours
156 'email': 60000 * 10, // 10 minutes 156 'email': 60000 * 10, // 10 minutes
157 'videos-views': undefined, // Unlimited 157 'videos-views': undefined, // Unlimited
158 'activitypub-refresher': 60000 * 10 // 10 minutes 158 'activitypub-refresher': 60000 * 10, // 10 minutes
159 'video-redundancy': 1000 * 3600 * 3 // 3 hours
159} 160}
160const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = { 161const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = {
161 'videos-views': { 162 'videos-views': {
diff --git a/server/initializers/migrations/0475-redundancy-expires-on.ts b/server/initializers/migrations/0475-redundancy-expires-on.ts
new file mode 100644
index 000000000..7e392c8c0
--- /dev/null
+++ b/server/initializers/migrations/0475-redundancy-expires-on.ts
@@ -0,0 +1,27 @@
1import * as Sequelize from 'sequelize'
2
3async function up (utils: {
4 transaction: Sequelize.Transaction,
5 queryInterface: Sequelize.QueryInterface,
6 sequelize: Sequelize.Sequelize,
7 db: any
8}): Promise<void> {
9 {
10 const data = {
11 type: Sequelize.DATE,
12 allowNull: true,
13 defaultValue: null
14 }
15
16 await utils.queryInterface.changeColumn('videoRedundancy', 'expiresOn', data)
17 }
18}
19
20function down (options) {
21 throw new Error('Not implemented.')
22}
23
24export {
25 up,
26 down
27}
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts
index 65b2dcb49..8252e95e9 100644
--- a/server/lib/activitypub/cache-file.ts
+++ b/server/lib/activitypub/cache-file.ts
@@ -13,7 +13,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
13 if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) 13 if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url)
14 14
15 return { 15 return {
16 expiresOn: new Date(cacheFileObject.expires), 16 expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null,
17 url: cacheFileObject.id, 17 url: cacheFileObject.id,
18 fileUrl: url.href, 18 fileUrl: url.href,
19 strategy: null, 19 strategy: null,
@@ -30,7 +30,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
30 if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`) 30 if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`)
31 31
32 return { 32 return {
33 expiresOn: new Date(cacheFileObject.expires), 33 expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null,
34 url: cacheFileObject.id, 34 url: cacheFileObject.id,
35 fileUrl: url.href, 35 fileUrl: url.href,
36 strategy: null, 36 strategy: null,
diff --git a/server/lib/job-queue/handlers/video-redundancy.ts b/server/lib/job-queue/handlers/video-redundancy.ts
new file mode 100644
index 000000000..319d7090e
--- /dev/null
+++ b/server/lib/job-queue/handlers/video-redundancy.ts
@@ -0,0 +1,20 @@
1import * as Bull from 'bull'
2import { logger } from '../../../helpers/logger'
3import { VideosRedundancyScheduler } from '@server/lib/schedulers/videos-redundancy-scheduler'
4
5export type VideoRedundancyPayload = {
6 videoId: number
7}
8
9async function processVideoRedundancy (job: Bull.Job) {
10 const payload = job.data as VideoRedundancyPayload
11 logger.info('Processing video redundancy in job %d.', job.id)
12
13 return VideosRedundancyScheduler.Instance.createManualRedundancy(payload.videoId)
14}
15
16// ---------------------------------------------------------------------------
17
18export {
19 processVideoRedundancy
20}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index ec601e9ea..8bbf58f2b 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -13,6 +13,7 @@ import { processVideoImport, VideoImportPayload } from './handlers/video-import'
13import { processVideosViews } from './handlers/video-views' 13import { processVideosViews } from './handlers/video-views'
14import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher' 14import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher'
15import { processVideoFileImport, VideoFileImportPayload } from './handlers/video-file-import' 15import { processVideoFileImport, VideoFileImportPayload } from './handlers/video-file-import'
16import { processVideoRedundancy, VideoRedundancyPayload } from '@server/lib/job-queue/handlers/video-redundancy'
16 17
17type CreateJobArgument = 18type CreateJobArgument =
18 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 19 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -24,20 +25,21 @@ type CreateJobArgument =
24 { type: 'email', payload: EmailPayload } | 25 { type: 'email', payload: EmailPayload } |
25 { type: 'video-import', payload: VideoImportPayload } | 26 { type: 'video-import', payload: VideoImportPayload } |
26 { type: 'activitypub-refresher', payload: RefreshPayload } | 27 { type: 'activitypub-refresher', payload: RefreshPayload } |
27 { type: 'videos-views', payload: {} } 28 { type: 'videos-views', payload: {} } |
29 { type: 'video-redundancy', payload: VideoRedundancyPayload }
28 30
29const handlers: { [ id in (JobType | 'video-file') ]: (job: Bull.Job) => Promise<any>} = { 31const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
30 'activitypub-http-broadcast': processActivityPubHttpBroadcast, 32 'activitypub-http-broadcast': processActivityPubHttpBroadcast,
31 'activitypub-http-unicast': processActivityPubHttpUnicast, 33 'activitypub-http-unicast': processActivityPubHttpUnicast,
32 'activitypub-http-fetcher': processActivityPubHttpFetcher, 34 'activitypub-http-fetcher': processActivityPubHttpFetcher,
33 'activitypub-follow': processActivityPubFollow, 35 'activitypub-follow': processActivityPubFollow,
34 'video-file-import': processVideoFileImport, 36 'video-file-import': processVideoFileImport,
35 'video-transcoding': processVideoTranscoding, 37 'video-transcoding': processVideoTranscoding,
36 'video-file': processVideoTranscoding, // TODO: remove it (changed in 1.3)
37 'email': processEmail, 38 'email': processEmail,
38 'video-import': processVideoImport, 39 'video-import': processVideoImport,
39 'videos-views': processVideosViews, 40 'videos-views': processVideosViews,
40 'activitypub-refresher': refreshAPObject 41 'activitypub-refresher': refreshAPObject,
42 'video-redundancy': processVideoRedundancy
41} 43}
42 44
43const jobTypes: JobType[] = [ 45const jobTypes: JobType[] = [
@@ -50,7 +52,8 @@ const jobTypes: JobType[] = [
50 'video-file-import', 52 'video-file-import',
51 'video-import', 53 'video-import',
52 'videos-views', 54 'videos-views',
53 'activitypub-refresher' 55 'activitypub-refresher',
56 'video-redundancy'
54] 57]
55 58
56class JobQueue { 59class JobQueue {
diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts
index 1b4ecd7c0..78d84e02e 100644
--- a/server/lib/redundancy.ts
+++ b/server/lib/redundancy.ts
@@ -13,10 +13,10 @@ async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t?
13 await videoRedundancy.destroy({ transaction: t }) 13 await videoRedundancy.destroy({ transaction: t })
14} 14}
15 15
16async function removeRedundancyOf (serverId: number) { 16async function removeRedundanciesOfServer (serverId: number) {
17 const videosRedundancy = await VideoRedundancyModel.listLocalOfServer(serverId) 17 const redundancies = await VideoRedundancyModel.listLocalOfServer(serverId)
18 18
19 for (const redundancy of videosRedundancy) { 19 for (const redundancy of redundancies) {
20 await removeVideoRedundancy(redundancy) 20 await removeVideoRedundancy(redundancy)
21 } 21 }
22} 22}
@@ -24,6 +24,6 @@ async function removeRedundancyOf (serverId: number) {
24// --------------------------------------------------------------------------- 24// ---------------------------------------------------------------------------
25 25
26export { 26export {
27 removeRedundancyOf, 27 removeRedundanciesOfServer,
28 removeVideoRedundancy 28 removeVideoRedundancy
29} 29}
diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts
index 350a335d3..956780a77 100644
--- a/server/lib/schedulers/update-videos-scheduler.ts
+++ b/server/lib/schedulers/update-videos-scheduler.ts
@@ -4,7 +4,6 @@ import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-upda
4import { retryTransactionWrapper } from '../../helpers/database-utils' 4import { retryTransactionWrapper } from '../../helpers/database-utils'
5import { federateVideoIfNeeded } from '../activitypub' 5import { federateVideoIfNeeded } from '../activitypub'
6import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' 6import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
7import { VideoPrivacy } from '../../../shared/models/videos'
8import { Notifier } from '../notifier' 7import { Notifier } from '../notifier'
9import { sequelizeTypescript } from '../../initializers/database' 8import { sequelizeTypescript } from '../../initializers/database'
10import { MVideoFullLight } from '@server/typings/models' 9import { MVideoFullLight } from '@server/typings/models'
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index c1c91b656..6e61cbe7d 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,7 +1,7 @@
1import { AbstractScheduler } from './abstract-scheduler' 1import { AbstractScheduler } from './abstract-scheduler'
2import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER } from '../../initializers/constants' 2import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER } from '../../initializers/constants'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { VideosRedundancy } from '../../../shared/models/redundancy' 4import { VideosRedundancyStrategy } from '../../../shared/models/redundancy'
5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 5import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
6import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent' 6import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent'
7import { join } from 'path' 7import { join } from 'path'
@@ -25,9 +25,10 @@ import {
25 MVideoWithAllFiles 25 MVideoWithAllFiles
26} from '@server/typings/models' 26} from '@server/typings/models'
27import { getVideoFilename } from '../video-paths' 27import { getVideoFilename } from '../video-paths'
28import { VideoModel } from '@server/models/video/video'
28 29
29type CandidateToDuplicate = { 30type CandidateToDuplicate = {
30 redundancy: VideosRedundancy, 31 redundancy: VideosRedundancyStrategy,
31 video: MVideoWithAllFiles, 32 video: MVideoWithAllFiles,
32 files: MVideoFile[], 33 files: MVideoFile[],
33 streamingPlaylists: MStreamingPlaylistFiles[] 34 streamingPlaylists: MStreamingPlaylistFiles[]
@@ -41,7 +42,7 @@ function isMVideoRedundancyFileVideo (
41 42
42export class VideosRedundancyScheduler extends AbstractScheduler { 43export class VideosRedundancyScheduler extends AbstractScheduler {
43 44
44 private static instance: AbstractScheduler 45 private static instance: VideosRedundancyScheduler
45 46
46 protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL 47 protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL
47 48
@@ -49,6 +50,22 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
49 super() 50 super()
50 } 51 }
51 52
53 async createManualRedundancy (videoId: number) {
54 const videoToDuplicate = await VideoModel.loadWithFiles(videoId)
55
56 if (!videoToDuplicate) {
57 logger.warn('Video to manually duplicate %d does not exist anymore.', videoId)
58 return
59 }
60
61 return this.createVideoRedundancies({
62 video: videoToDuplicate,
63 redundancy: null,
64 files: videoToDuplicate.VideoFiles,
65 streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists
66 })
67 }
68
52 protected async internalExecute () { 69 protected async internalExecute () {
53 for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { 70 for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
54 logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy) 71 logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy)
@@ -94,7 +111,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
94 for (const redundancyModel of expired) { 111 for (const redundancyModel of expired) {
95 try { 112 try {
96 const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) 113 const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
97 const candidate = { 114 const candidate: CandidateToDuplicate = {
98 redundancy: redundancyConfig, 115 redundancy: redundancyConfig,
99 video: null, 116 video: null,
100 files: [], 117 files: [],
@@ -140,7 +157,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
140 } 157 }
141 } 158 }
142 159
143 private findVideoToDuplicate (cache: VideosRedundancy) { 160 private findVideoToDuplicate (cache: VideosRedundancyStrategy) {
144 if (cache.strategy === 'most-views') { 161 if (cache.strategy === 'most-views') {
145 return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) 162 return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
146 } 163 }
@@ -187,13 +204,21 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
187 } 204 }
188 } 205 }
189 206
190 private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: MVideoAccountLight, fileArg: MVideoFile) { 207 private async createVideoFileRedundancy (redundancy: VideosRedundancyStrategy | null, video: MVideoAccountLight, fileArg: MVideoFile) {
208 let strategy = 'manual'
209 let expiresOn: Date = null
210
211 if (redundancy) {
212 strategy = redundancy.strategy
213 expiresOn = this.buildNewExpiration(redundancy.minLifetime)
214 }
215
191 const file = fileArg as MVideoFileVideo 216 const file = fileArg as MVideoFileVideo
192 file.Video = video 217 file.Video = video
193 218
194 const serverActor = await getServerActor() 219 const serverActor = await getServerActor()
195 220
196 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) 221 logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy)
197 222
198 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() 223 const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
199 const magnetUri = generateMagnetUri(video, file, baseUrlHttp, baseUrlWs) 224 const magnetUri = generateMagnetUri(video, file, baseUrlHttp, baseUrlWs)
@@ -204,10 +229,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
204 await move(tmpPath, destPath, { overwrite: true }) 229 await move(tmpPath, destPath, { overwrite: true })
205 230
206 const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ 231 const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({
207 expiresOn: this.buildNewExpiration(redundancy.minLifetime), 232 expiresOn,
208 url: getVideoCacheFileActivityPubUrl(file), 233 url: getVideoCacheFileActivityPubUrl(file),
209 fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL), 234 fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL),
210 strategy: redundancy.strategy, 235 strategy,
211 videoFileId: file.id, 236 videoFileId: file.id,
212 actorId: serverActor.id 237 actorId: serverActor.id
213 }) 238 })
@@ -220,25 +245,33 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
220 } 245 }
221 246
222 private async createStreamingPlaylistRedundancy ( 247 private async createStreamingPlaylistRedundancy (
223 redundancy: VideosRedundancy, 248 redundancy: VideosRedundancyStrategy,
224 video: MVideoAccountLight, 249 video: MVideoAccountLight,
225 playlistArg: MStreamingPlaylist 250 playlistArg: MStreamingPlaylist
226 ) { 251 ) {
252 let strategy = 'manual'
253 let expiresOn: Date = null
254
255 if (redundancy) {
256 strategy = redundancy.strategy
257 expiresOn = this.buildNewExpiration(redundancy.minLifetime)
258 }
259
227 const playlist = playlistArg as MStreamingPlaylistVideo 260 const playlist = playlistArg as MStreamingPlaylistVideo
228 playlist.Video = video 261 playlist.Video = video
229 262
230 const serverActor = await getServerActor() 263 const serverActor = await getServerActor()
231 264
232 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy) 265 logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy)
233 266
234 const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) 267 const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
235 await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) 268 await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
236 269
237 const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ 270 const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({
238 expiresOn: this.buildNewExpiration(redundancy.minLifetime), 271 expiresOn,
239 url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), 272 url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist),
240 fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL), 273 fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL),
241 strategy: redundancy.strategy, 274 strategy,
242 videoStreamingPlaylistId: playlist.id, 275 videoStreamingPlaylistId: playlist.id,
243 actorId: serverActor.id 276 actorId: serverActor.id
244 }) 277 })
diff --git a/server/middlewares/sort.ts b/server/middlewares/sort.ts
index 8c27e8237..75238228f 100644
--- a/server/middlewares/sort.ts
+++ b/server/middlewares/sort.ts
@@ -1,17 +1,11 @@
1import * as express from 'express' 1import * as express from 'express'
2import { SortType } from '../models/utils' 2import { SortType } from '../models/utils'
3 3
4function setDefaultSort (req: express.Request, res: express.Response, next: express.NextFunction) { 4const setDefaultSort = setDefaultSortFactory('-createdAt')
5 if (!req.query.sort) req.query.sort = '-createdAt'
6
7 return next()
8}
9 5
10function setDefaultSearchSort (req: express.Request, res: express.Response, next: express.NextFunction) { 6const setDefaultVideoRedundanciesSort = setDefaultSortFactory('name')
11 if (!req.query.sort) req.query.sort = '-match'
12 7
13 return next() 8const setDefaultSearchSort = setDefaultSortFactory('-match')
14}
15 9
16function setBlacklistSort (req: express.Request, res: express.Response, next: express.NextFunction) { 10function setBlacklistSort (req: express.Request, res: express.Response, next: express.NextFunction) {
17 let newSort: SortType = { sortModel: undefined, sortValue: '' } 11 let newSort: SortType = { sortModel: undefined, sortValue: '' }
@@ -39,5 +33,16 @@ function setBlacklistSort (req: express.Request, res: express.Response, next: ex
39export { 33export {
40 setDefaultSort, 34 setDefaultSort,
41 setDefaultSearchSort, 35 setDefaultSearchSort,
36 setDefaultVideoRedundanciesSort,
42 setBlacklistSort 37 setBlacklistSort
43} 38}
39
40// ---------------------------------------------------------------------------
41
42function setDefaultSortFactory (sort: string) {
43 return (req: express.Request, res: express.Response, next: express.NextFunction) => {
44 if (!req.query.sort) req.query.sort = sort
45
46 return next()
47 }
48}
diff --git a/server/middlewares/validators/redundancy.ts b/server/middlewares/validators/redundancy.ts
index 8098e3a44..16b42fc0d 100644
--- a/server/middlewares/validators/redundancy.ts
+++ b/server/middlewares/validators/redundancy.ts
@@ -1,12 +1,13 @@
1import * as express from 'express' 1import * as express from 'express'
2import { body, param } from 'express-validator' 2import { body, param, query } from 'express-validator'
3import { exists, isBooleanValid, isIdOrUUIDValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc' 3import { exists, isBooleanValid, isIdOrUUIDValid, isIdValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc'
4import { logger } from '../../helpers/logger' 4import { logger } from '../../helpers/logger'
5import { areValidationErrors } from './utils' 5import { areValidationErrors } from './utils'
6import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' 6import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
7import { isHostValid } from '../../helpers/custom-validators/servers' 7import { isHostValid } from '../../helpers/custom-validators/servers'
8import { ServerModel } from '../../models/server/server' 8import { ServerModel } from '../../models/server/server'
9import { doesVideoExist } from '../../helpers/middlewares' 9import { doesVideoExist } from '../../helpers/middlewares'
10import { isVideoRedundancyTarget } from '@server/helpers/custom-validators/video-redundancies'
10 11
11const videoFileRedundancyGetValidator = [ 12const videoFileRedundancyGetValidator = [
12 param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), 13 param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'),
@@ -101,10 +102,77 @@ const updateServerRedundancyValidator = [
101 } 102 }
102] 103]
103 104
105const listVideoRedundanciesValidator = [
106 query('target')
107 .custom(isVideoRedundancyTarget).withMessage('Should have a valid video redundancies target'),
108
109 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
110 logger.debug('Checking listVideoRedundanciesValidator parameters', { parameters: req.query })
111
112 if (areValidationErrors(req, res)) return
113
114 return next()
115 }
116]
117
118const addVideoRedundancyValidator = [
119 body('videoId')
120 .custom(isIdValid)
121 .withMessage('Should have a valid video id'),
122
123 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
124 logger.debug('Checking addVideoRedundancyValidator parameters', { parameters: req.query })
125
126 if (areValidationErrors(req, res)) return
127
128 if (!await doesVideoExist(req.body.videoId, res, 'only-video')) return
129
130 if (res.locals.onlyVideo.remote === false) {
131 return res.status(400)
132 .json({ error: 'Cannot create a redundancy on a local video' })
133 .end()
134 }
135
136 const alreadyExists = await VideoRedundancyModel.isLocalByVideoUUIDExists(res.locals.onlyVideo.uuid)
137 if (alreadyExists) {
138 return res.status(409)
139 .json({ error: 'This video is already duplicated by your instance.' })
140 }
141
142 return next()
143 }
144]
145
146const removeVideoRedundancyValidator = [
147 param('redundancyId')
148 .custom(isIdValid)
149 .withMessage('Should have a valid redundancy id'),
150
151 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
152 logger.debug('Checking removeVideoRedundancyValidator parameters', { parameters: req.query })
153
154 if (areValidationErrors(req, res)) return
155
156 const redundancy = await VideoRedundancyModel.loadByIdWithVideo(parseInt(req.params.redundancyId, 10))
157 if (!redundancy) {
158 return res.status(404)
159 .json({ error: 'Video redundancy not found' })
160 .end()
161 }
162
163 res.locals.videoRedundancy = redundancy
164
165 return next()
166 }
167]
168
104// --------------------------------------------------------------------------- 169// ---------------------------------------------------------------------------
105 170
106export { 171export {
107 videoFileRedundancyGetValidator, 172 videoFileRedundancyGetValidator,
108 videoPlaylistRedundancyGetValidator, 173 videoPlaylistRedundancyGetValidator,
109 updateServerRedundancyValidator 174 updateServerRedundancyValidator,
175 listVideoRedundanciesValidator,
176 addVideoRedundancyValidator,
177 removeVideoRedundancyValidator
110} 178}
diff --git a/server/middlewares/validators/sort.ts b/server/middlewares/validators/sort.ts
index c75e701d6..b76dab722 100644
--- a/server/middlewares/validators/sort.ts
+++ b/server/middlewares/validators/sort.ts
@@ -23,6 +23,7 @@ const SORTABLE_USER_NOTIFICATIONS_COLUMNS = createSortableColumns(SORTABLE_COLUM
23const SORTABLE_VIDEO_PLAYLISTS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_PLAYLISTS) 23const SORTABLE_VIDEO_PLAYLISTS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_PLAYLISTS)
24const SORTABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.PLUGINS) 24const SORTABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.PLUGINS)
25const SORTABLE_AVAILABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.AVAILABLE_PLUGINS) 25const SORTABLE_AVAILABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.AVAILABLE_PLUGINS)
26const SORTABLE_VIDEO_REDUNDANCIES_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES)
26 27
27const usersSortValidator = checkSort(SORTABLE_USERS_COLUMNS) 28const usersSortValidator = checkSort(SORTABLE_USERS_COLUMNS)
28const accountsSortValidator = checkSort(SORTABLE_ACCOUNTS_COLUMNS) 29const accountsSortValidator = checkSort(SORTABLE_ACCOUNTS_COLUMNS)
@@ -45,6 +46,7 @@ const userNotificationsSortValidator = checkSort(SORTABLE_USER_NOTIFICATIONS_COL
45const videoPlaylistsSortValidator = checkSort(SORTABLE_VIDEO_PLAYLISTS_COLUMNS) 46const videoPlaylistsSortValidator = checkSort(SORTABLE_VIDEO_PLAYLISTS_COLUMNS)
46const pluginsSortValidator = checkSort(SORTABLE_PLUGINS_COLUMNS) 47const pluginsSortValidator = checkSort(SORTABLE_PLUGINS_COLUMNS)
47const availablePluginsSortValidator = checkSort(SORTABLE_AVAILABLE_PLUGINS_COLUMNS) 48const availablePluginsSortValidator = checkSort(SORTABLE_AVAILABLE_PLUGINS_COLUMNS)
49const videoRedundanciesSortValidator = checkSort(SORTABLE_VIDEO_REDUNDANCIES_COLUMNS)
48 50
49// --------------------------------------------------------------------------- 51// ---------------------------------------------------------------------------
50 52
@@ -69,5 +71,6 @@ export {
69 serversBlocklistSortValidator, 71 serversBlocklistSortValidator,
70 userNotificationsSortValidator, 72 userNotificationsSortValidator,
71 videoPlaylistsSortValidator, 73 videoPlaylistsSortValidator,
74 videoRedundanciesSortValidator,
72 pluginsSortValidator 75 pluginsSortValidator
73} 76}
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index 8c9a7eabf..4e66d72e3 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -13,13 +13,13 @@ import {
13 UpdatedAt 13 UpdatedAt
14} from 'sequelize-typescript' 14} from 'sequelize-typescript'
15import { ActorModel } from '../activitypub/actor' 15import { ActorModel } from '../activitypub/actor'
16import { getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils' 16import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
17import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc' 17import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
18import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants' 18import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
19import { VideoFileModel } from '../video/video-file' 19import { VideoFileModel } from '../video/video-file'
20import { getServerActor } from '../../helpers/utils' 20import { getServerActor } from '../../helpers/utils'
21import { VideoModel } from '../video/video' 21import { VideoModel } from '../video/video'
22import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' 22import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy'
23import { logger } from '../../helpers/logger' 23import { logger } from '../../helpers/logger'
24import { CacheFileObject, VideoPrivacy } from '../../../shared' 24import { CacheFileObject, VideoPrivacy } from '../../../shared'
25import { VideoChannelModel } from '../video/video-channel' 25import { VideoChannelModel } from '../video/video-channel'
@@ -27,10 +27,16 @@ import { ServerModel } from '../server/server'
27import { sample } from 'lodash' 27import { sample } from 'lodash'
28import { isTestInstance } from '../../helpers/core-utils' 28import { isTestInstance } from '../../helpers/core-utils'
29import * as Bluebird from 'bluebird' 29import * as Bluebird from 'bluebird'
30import { col, FindOptions, fn, literal, Op, Transaction } from 'sequelize' 30import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize'
31import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist' 31import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
32import { CONFIG } from '../../initializers/config' 32import { CONFIG } from '../../initializers/config'
33import { MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/typings/models' 33import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/typings/models'
34import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
35import {
36 FileRedundancyInformation,
37 StreamingPlaylistRedundancyInformation,
38 VideoRedundancy
39} from '@shared/models/redundancy/video-redundancy.model'
34 40
35export enum ScopeNames { 41export enum ScopeNames {
36 WITH_VIDEO = 'WITH_VIDEO' 42 WITH_VIDEO = 'WITH_VIDEO'
@@ -86,7 +92,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
86 @UpdatedAt 92 @UpdatedAt
87 updatedAt: Date 93 updatedAt: Date
88 94
89 @AllowNull(false) 95 @AllowNull(true)
90 @Column 96 @Column
91 expiresOn: Date 97 expiresOn: Date
92 98
@@ -193,6 +199,15 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
193 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) 199 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
194 } 200 }
195 201
202 static loadByIdWithVideo (id: number, transaction?: Transaction): Bluebird<MVideoRedundancyVideo> {
203 const query = {
204 where: { id },
205 transaction
206 }
207
208 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
209 }
210
196 static loadByUrl (url: string, transaction?: Transaction): Bluebird<MVideoRedundancy> { 211 static loadByUrl (url: string, transaction?: Transaction): Bluebird<MVideoRedundancy> {
197 const query = { 212 const query = {
198 where: { 213 where: {
@@ -394,7 +409,8 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
394 [Op.ne]: actor.id 409 [Op.ne]: actor.id
395 }, 410 },
396 expiresOn: { 411 expiresOn: {
397 [ Op.lt ]: new Date() 412 [ Op.lt ]: new Date(),
413 [ Op.ne ]: null
398 } 414 }
399 } 415 }
400 } 416 }
@@ -447,7 +463,112 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
447 return VideoRedundancyModel.findAll(query) 463 return VideoRedundancyModel.findAll(query)
448 } 464 }
449 465
450 static async getStats (strategy: VideoRedundancyStrategy) { 466 static listForApi (options: {
467 start: number,
468 count: number,
469 sort: string,
470 target: VideoRedundanciesTarget,
471 strategy?: string
472 }) {
473 const { start, count, sort, target, strategy } = options
474 let redundancyWhere: WhereOptions = {}
475 let videosWhere: WhereOptions = {}
476 let redundancySqlSuffix = ''
477
478 if (target === 'my-videos') {
479 Object.assign(videosWhere, { remote: false })
480 } else if (target === 'remote-videos') {
481 Object.assign(videosWhere, { remote: true })
482 Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
483 redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
484 }
485
486 if (strategy) {
487 Object.assign(redundancyWhere, { strategy: strategy })
488 }
489
490 const videoFilterWhere = {
491 [Op.and]: [
492 {
493 [ Op.or ]: [
494 {
495 id: {
496 [ Op.in ]: literal(
497 '(' +
498 'SELECT "videoId" FROM "videoFile" ' +
499 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
500 redundancySqlSuffix +
501 ')'
502 )
503 }
504 },
505 {
506 id: {
507 [ Op.in ]: literal(
508 '(' +
509 'select "videoId" FROM "videoStreamingPlaylist" ' +
510 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
511 redundancySqlSuffix +
512 ')'
513 )
514 }
515 }
516 ]
517 },
518
519 videosWhere
520 ]
521 }
522
523 // /!\ On video model /!\
524 const findOptions = {
525 offset: start,
526 limit: count,
527 order: getSort(sort),
528 include: [
529 {
530 required: false,
531 model: VideoFileModel.unscoped(),
532 include: [
533 {
534 model: VideoRedundancyModel.unscoped(),
535 required: false,
536 where: redundancyWhere
537 }
538 ]
539 },
540 {
541 required: false,
542 model: VideoStreamingPlaylistModel.unscoped(),
543 include: [
544 {
545 model: VideoRedundancyModel.unscoped(),
546 required: false,
547 where: redundancyWhere
548 },
549 {
550 model: VideoFileModel.unscoped(),
551 required: false
552 }
553 ]
554 }
555 ],
556 where: videoFilterWhere
557 }
558
559 // /!\ On video model /!\
560 const countOptions = {
561 where: videoFilterWhere
562 }
563
564 return Promise.all([
565 VideoModel.findAll(findOptions),
566
567 VideoModel.count(countOptions)
568 ]).then(([ data, total ]) => ({ total, data }))
569 }
570
571 static async getStats (strategy: VideoRedundancyStrategyWithManual) {
451 const actor = await getServerActor() 572 const actor = await getServerActor()
452 573
453 const query: FindOptions = { 574 const query: FindOptions = {
@@ -478,6 +599,53 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
478 })) 599 }))
479 } 600 }
480 601
602 static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
603 let filesRedundancies: FileRedundancyInformation[] = []
604 let streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = []
605
606 for (const file of video.VideoFiles) {
607 for (const redundancy of file.RedundancyVideos) {
608 filesRedundancies.push({
609 id: redundancy.id,
610 fileUrl: redundancy.fileUrl,
611 strategy: redundancy.strategy,
612 createdAt: redundancy.createdAt,
613 updatedAt: redundancy.updatedAt,
614 expiresOn: redundancy.expiresOn,
615 size: file.size
616 })
617 }
618 }
619
620 for (const playlist of video.VideoStreamingPlaylists) {
621 const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0)
622
623 for (const redundancy of playlist.RedundancyVideos) {
624 streamingPlaylistsRedundancies.push({
625 id: redundancy.id,
626 fileUrl: redundancy.fileUrl,
627 strategy: redundancy.strategy,
628 createdAt: redundancy.createdAt,
629 updatedAt: redundancy.updatedAt,
630 expiresOn: redundancy.expiresOn,
631 size
632 })
633 }
634 }
635
636 return {
637 id: video.id,
638 name: video.name,
639 url: video.url,
640 uuid: video.uuid,
641
642 redundancies: {
643 files: filesRedundancies,
644 streamingPlaylists: streamingPlaylistsRedundancies
645 }
646 }
647 }
648
481 getVideo () { 649 getVideo () {
482 if (this.VideoFile) return this.VideoFile.Video 650 if (this.VideoFile) return this.VideoFile.Video
483 651
@@ -494,7 +662,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
494 id: this.url, 662 id: this.url,
495 type: 'CacheFile' as 'CacheFile', 663 type: 'CacheFile' as 'CacheFile',
496 object: this.VideoStreamingPlaylist.Video.url, 664 object: this.VideoStreamingPlaylist.Video.url,
497 expires: this.expiresOn.toISOString(), 665 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
498 url: { 666 url: {
499 type: 'Link', 667 type: 'Link',
500 mediaType: 'application/x-mpegURL', 668 mediaType: 'application/x-mpegURL',
@@ -507,7 +675,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
507 id: this.url, 675 id: this.url,
508 type: 'CacheFile' as 'CacheFile', 676 type: 'CacheFile' as 'CacheFile',
509 object: this.VideoFile.Video.url, 677 object: this.VideoFile.Video.url,
510 expires: this.expiresOn.toISOString(), 678 expires: this.expiresOn ? this.expiresOn.toISOString() : null,
511 url: { 679 url: {
512 type: 'Link', 680 type: 'Link',
513 mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any, 681 mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
diff --git a/server/tests/api/check-params/redundancy.ts b/server/tests/api/check-params/redundancy.ts
index 6471da840..7012a39ee 100644
--- a/server/tests/api/check-params/redundancy.ts
+++ b/server/tests/api/check-params/redundancy.ts
@@ -3,21 +3,25 @@
3import 'mocha' 3import 'mocha'
4 4
5import { 5import {
6 checkBadCountPagination,
7 checkBadSortPagination,
8 checkBadStartPagination,
6 cleanupTests, 9 cleanupTests,
7 createUser, 10 createUser,
8 doubleFollow, 11 doubleFollow,
9 flushAndRunMultipleServers, 12 flushAndRunMultipleServers, makeDeleteRequest,
10 flushTests, 13 makeGetRequest, makePostBodyRequest,
11 killallServers,
12 makePutBodyRequest, 14 makePutBodyRequest,
13 ServerInfo, 15 ServerInfo,
14 setAccessTokensToServers, 16 setAccessTokensToServers, uploadVideoAndGetId,
15 userLogin 17 userLogin, waitJobs
16} from '../../../../shared/extra-utils' 18} from '../../../../shared/extra-utils'
17 19
18describe('Test server redundancy API validators', function () { 20describe('Test server redundancy API validators', function () {
19 let servers: ServerInfo[] 21 let servers: ServerInfo[]
20 let userAccessToken = null 22 let userAccessToken = null
23 let videoIdLocal: number
24 let videoIdRemote: number
21 25
22 // --------------------------------------------------------------- 26 // ---------------------------------------------------------------
23 27
@@ -36,9 +40,134 @@ describe('Test server redundancy API validators', function () {
36 40
37 await createUser({ url: servers[ 0 ].url, accessToken: servers[ 0 ].accessToken, username: user.username, password: user.password }) 41 await createUser({ url: servers[ 0 ].url, accessToken: servers[ 0 ].accessToken, username: user.username, password: user.password })
38 userAccessToken = await userLogin(servers[0], user) 42 userAccessToken = await userLogin(servers[0], user)
43
44 videoIdLocal = (await uploadVideoAndGetId({ server: servers[0], videoName: 'video' })).id
45 videoIdRemote = (await uploadVideoAndGetId({ server: servers[1], videoName: 'video' })).id
46
47 await waitJobs(servers)
48 })
49
50 describe('When listing redundancies', function () {
51 const path = '/api/v1/server/redundancy/videos'
52
53 let url: string
54 let token: string
55
56 before(function () {
57 url = servers[0].url
58 token = servers[0].accessToken
59 })
60
61 it('Should fail with an invalid token', async function () {
62 await makeGetRequest({ url, path, token: 'fake_token', statusCodeExpected: 401 })
63 })
64
65 it('Should fail if the user is not an administrator', async function () {
66 await makeGetRequest({ url, path, token: userAccessToken, statusCodeExpected: 403 })
67 })
68
69 it('Should fail with a bad start pagination', async function () {
70 await checkBadStartPagination(url, path, servers[0].accessToken)
71 })
72
73 it('Should fail with a bad count pagination', async function () {
74 await checkBadCountPagination(url, path, servers[0].accessToken)
75 })
76
77 it('Should fail with an incorrect sort', async function () {
78 await checkBadSortPagination(url, path, servers[0].accessToken)
79 })
80
81 it('Should fail with a bad target', async function () {
82 await makeGetRequest({ url, path, token, query: { target: 'bad target' } })
83 })
84
85 it('Should fail without target', async function () {
86 await makeGetRequest({ url, path, token })
87 })
88
89 it('Should succeed with the correct params', async function () {
90 await makeGetRequest({ url, path, token, query: { target: 'my-videos' }, statusCodeExpected: 200 })
91 })
92 })
93
94 describe('When manually adding a redundancy', function () {
95 const path = '/api/v1/server/redundancy/videos'
96
97 let url: string
98 let token: string
99
100 before(function () {
101 url = servers[0].url
102 token = servers[0].accessToken
103 })
104
105 it('Should fail with an invalid token', async function () {
106 await makePostBodyRequest({ url, path, token: 'fake_token', statusCodeExpected: 401 })
107 })
108
109 it('Should fail if the user is not an administrator', async function () {
110 await makePostBodyRequest({ url, path, token: userAccessToken, statusCodeExpected: 403 })
111 })
112
113 it('Should fail without a video id', async function () {
114 await makePostBodyRequest({ url, path, token })
115 })
116
117 it('Should fail with an incorrect video id', async function () {
118 await makePostBodyRequest({ url, path, token, fields: { videoId: 'peertube' } })
119 })
120
121 it('Should fail with a not found video id', async function () {
122 await makePostBodyRequest({ url, path, token, fields: { videoId: 6565 }, statusCodeExpected: 404 })
123 })
124
125 it('Should fail with a local a video id', async function () {
126 await makePostBodyRequest({ url, path, token, fields: { videoId: videoIdLocal } })
127 })
128
129 it('Should succeed with the correct params', async function () {
130 await makePostBodyRequest({ url, path, token, fields: { videoId: videoIdRemote }, statusCodeExpected: 204 })
131 })
132
133 it('Should fail if the video is already duplicated', async function () {
134 this.timeout(30000)
135
136 await waitJobs(servers)
137
138 await makePostBodyRequest({ url, path, token, fields: { videoId: videoIdRemote }, statusCodeExpected: 409 })
139 })
140 })
141
142 describe('When manually removing a redundancy', function () {
143 const path = '/api/v1/server/redundancy/videos/'
144
145 let url: string
146 let token: string
147
148 before(function () {
149 url = servers[0].url
150 token = servers[0].accessToken
151 })
152
153 it('Should fail with an invalid token', async function () {
154 await makeDeleteRequest({ url, path: path + '1', token: 'fake_token', statusCodeExpected: 401 })
155 })
156
157 it('Should fail if the user is not an administrator', async function () {
158 await makeDeleteRequest({ url, path: path + '1', token: userAccessToken, statusCodeExpected: 403 })
159 })
160
161 it('Should fail with an incorrect video id', async function () {
162 await makeDeleteRequest({ url, path: path + 'toto', token })
163 })
164
165 it('Should fail with a not found video redundancy', async function () {
166 await makeDeleteRequest({ url, path: path + '454545', token, statusCodeExpected: 404 })
167 })
39 }) 168 })
40 169
41 describe('When updating redundancy', function () { 170 describe('When updating server redundancy', function () {
42 const path = '/api/v1/server/redundancy' 171 const path = '/api/v1/server/redundancy'
43 172
44 it('Should fail with an invalid token', async function () { 173 it('Should fail with an invalid token', async function () {
diff --git a/server/tests/api/redundancy/index.ts b/server/tests/api/redundancy/index.ts
index 8e69b95a6..5359055b0 100644
--- a/server/tests/api/redundancy/index.ts
+++ b/server/tests/api/redundancy/index.ts
@@ -1 +1,2 @@
1import './redundancy' 1import './redundancy'
2import './manage-redundancy'
diff --git a/server/tests/api/redundancy/manage-redundancy.ts b/server/tests/api/redundancy/manage-redundancy.ts
new file mode 100644
index 000000000..6a8937f24
--- /dev/null
+++ b/server/tests/api/redundancy/manage-redundancy.ts
@@ -0,0 +1,373 @@
1/* tslint:disable:no-unused-expression */
2
3import * as chai from 'chai'
4import 'mocha'
5import {
6 cleanupTests,
7 doubleFollow,
8 flushAndRunMultipleServers,
9 getLocalIdByUUID,
10 ServerInfo,
11 setAccessTokensToServers,
12 uploadVideo,
13 uploadVideoAndGetId,
14 waitUntilLog
15} from '../../../../shared/extra-utils'
16import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
17import { addVideoRedundancy, listVideoRedundancies, removeVideoRedundancy, updateRedundancy } from '@shared/extra-utils/server/redundancy'
18import { VideoPrivacy, VideoRedundanciesTarget, VideoRedundancy } from '@shared/models'
19
20const expect = chai.expect
21
22describe('Test manage videos redundancy', function () {
23 const targets: VideoRedundanciesTarget[] = [ 'my-videos', 'remote-videos' ]
24
25 let servers: ServerInfo[]
26 let video1Server2UUID: string
27 let video2Server2UUID: string
28 let redundanciesToRemove: number[] = []
29
30 before(async function () {
31 this.timeout(120000)
32
33 const config = {
34 transcoding: {
35 hls: {
36 enabled: true
37 }
38 },
39 redundancy: {
40 videos: {
41 check_interval: '1 second',
42 strategies: [
43 {
44 strategy: 'recently-added',
45 min_lifetime: '1 hour',
46 size: '10MB',
47 min_views: 0
48 }
49 ]
50 }
51 }
52 }
53 servers = await flushAndRunMultipleServers(3, config)
54
55 // Get the access tokens
56 await setAccessTokensToServers(servers)
57
58 {
59 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
60 video1Server2UUID = res.body.video.uuid
61 }
62
63 {
64 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
65 video2Server2UUID = res.body.video.uuid
66 }
67
68 await waitJobs(servers)
69
70 // Server 1 and server 2 follow each other
71 await doubleFollow(servers[ 0 ], servers[ 1 ])
72 await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
73
74 await waitJobs(servers)
75 })
76
77 it('Should not have redundancies on server 3', async function () {
78 for (const target of targets) {
79 const res = await listVideoRedundancies({
80 url: servers[2].url,
81 accessToken: servers[2].accessToken,
82 target
83 })
84
85 expect(res.body.total).to.equal(0)
86 expect(res.body.data).to.have.lengthOf(0)
87 }
88 })
89
90 it('Should not have "remote-videos" redundancies on server 2', async function () {
91 this.timeout(120000)
92
93 await waitJobs(servers)
94 await waitUntilLog(servers[0], 'Duplicated ', 10)
95 await waitJobs(servers)
96
97 const res = await listVideoRedundancies({
98 url: servers[1].url,
99 accessToken: servers[1].accessToken,
100 target: 'remote-videos'
101 })
102
103 expect(res.body.total).to.equal(0)
104 expect(res.body.data).to.have.lengthOf(0)
105 })
106
107 it('Should have "my-videos" redundancies on server 2', async function () {
108 this.timeout(120000)
109
110 const res = await listVideoRedundancies({
111 url: servers[1].url,
112 accessToken: servers[1].accessToken,
113 target: 'my-videos'
114 })
115
116 expect(res.body.total).to.equal(2)
117
118 const videos = res.body.data as VideoRedundancy[]
119 expect(videos).to.have.lengthOf(2)
120
121 const videos1 = videos.find(v => v.uuid === video1Server2UUID)
122 const videos2 = videos.find(v => v.uuid === video2Server2UUID)
123
124 expect(videos1.name).to.equal('video 1 server 2')
125 expect(videos2.name).to.equal('video 2 server 2')
126
127 expect(videos1.redundancies.files).to.have.lengthOf(4)
128 expect(videos1.redundancies.streamingPlaylists).to.have.lengthOf(1)
129
130 const redundancies = videos1.redundancies.files.concat(videos1.redundancies.streamingPlaylists)
131
132 for (const r of redundancies) {
133 expect(r.strategy).to.be.null
134 expect(r.fileUrl).to.exist
135 expect(r.createdAt).to.exist
136 expect(r.updatedAt).to.exist
137 expect(r.expiresOn).to.exist
138 }
139 })
140
141 it('Should not have "my-videos" redundancies on server 1', async function () {
142 const res = await listVideoRedundancies({
143 url: servers[0].url,
144 accessToken: servers[0].accessToken,
145 target: 'my-videos'
146 })
147
148 expect(res.body.total).to.equal(0)
149 expect(res.body.data).to.have.lengthOf(0)
150 })
151
152 it('Should have "remote-videos" redundancies on server 1', async function () {
153 this.timeout(120000)
154
155 const res = await listVideoRedundancies({
156 url: servers[0].url,
157 accessToken: servers[0].accessToken,
158 target: 'remote-videos'
159 })
160
161 expect(res.body.total).to.equal(2)
162
163 const videos = res.body.data as VideoRedundancy[]
164 expect(videos).to.have.lengthOf(2)
165
166 const videos1 = videos.find(v => v.uuid === video1Server2UUID)
167 const videos2 = videos.find(v => v.uuid === video2Server2UUID)
168
169 expect(videos1.name).to.equal('video 1 server 2')
170 expect(videos2.name).to.equal('video 2 server 2')
171
172 expect(videos1.redundancies.files).to.have.lengthOf(4)
173 expect(videos1.redundancies.streamingPlaylists).to.have.lengthOf(1)
174
175 const redundancies = videos1.redundancies.files.concat(videos1.redundancies.streamingPlaylists)
176
177 for (const r of redundancies) {
178 expect(r.strategy).to.equal('recently-added')
179 expect(r.fileUrl).to.exist
180 expect(r.createdAt).to.exist
181 expect(r.updatedAt).to.exist
182 expect(r.expiresOn).to.exist
183 }
184 })
185
186 it('Should correctly paginate and sort results', async function () {
187 {
188 const res = await listVideoRedundancies({
189 url: servers[0].url,
190 accessToken: servers[0].accessToken,
191 target: 'remote-videos',
192 sort: 'name',
193 start: 0,
194 count: 2
195 })
196
197 const videos = res.body.data
198 expect(videos[ 0 ].name).to.equal('video 1 server 2')
199 expect(videos[ 1 ].name).to.equal('video 2 server 2')
200 }
201
202 {
203 const res = await listVideoRedundancies({
204 url: servers[0].url,
205 accessToken: servers[0].accessToken,
206 target: 'remote-videos',
207 sort: '-name',
208 start: 0,
209 count: 2
210 })
211
212 const videos = res.body.data
213 expect(videos[ 0 ].name).to.equal('video 2 server 2')
214 expect(videos[ 1 ].name).to.equal('video 1 server 2')
215 }
216
217 {
218 const res = await listVideoRedundancies({
219 url: servers[0].url,
220 accessToken: servers[0].accessToken,
221 target: 'remote-videos',
222 sort: '-name',
223 start: 1,
224 count: 1
225 })
226
227 const videos = res.body.data
228 expect(videos[ 0 ].name).to.equal('video 1 server 2')
229 }
230 })
231
232 it('Should manually add a redundancy and list it', async function () {
233 this.timeout(120000)
234
235 const uuid = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video 3 server 2', privacy: VideoPrivacy.UNLISTED })).uuid
236 await waitJobs(servers)
237 const videoId = await getLocalIdByUUID(servers[0].url, uuid)
238
239 await addVideoRedundancy({
240 url: servers[0].url,
241 accessToken: servers[0].accessToken,
242 videoId
243 })
244
245 await waitJobs(servers)
246 await waitUntilLog(servers[0], 'Duplicated ', 15)
247 await waitJobs(servers)
248
249 {
250 const res = await listVideoRedundancies({
251 url: servers[0].url,
252 accessToken: servers[0].accessToken,
253 target: 'remote-videos',
254 sort: '-name',
255 start: 0,
256 count: 5
257 })
258
259 const videos = res.body.data
260 expect(videos[ 0 ].name).to.equal('video 3 server 2')
261
262 const video = videos[ 0 ]
263 expect(video.redundancies.files).to.have.lengthOf(4)
264 expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1)
265
266 const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists)
267
268 for (const r of redundancies) {
269 redundanciesToRemove.push(r.id)
270
271 expect(r.strategy).to.equal('manual')
272 expect(r.fileUrl).to.exist
273 expect(r.createdAt).to.exist
274 expect(r.updatedAt).to.exist
275 expect(r.expiresOn).to.be.null
276 }
277 }
278
279 const res = await listVideoRedundancies({
280 url: servers[1].url,
281 accessToken: servers[1].accessToken,
282 target: 'my-videos',
283 sort: '-name',
284 start: 0,
285 count: 5
286 })
287
288 const videos = res.body.data
289 expect(videos[ 0 ].name).to.equal('video 3 server 2')
290
291 const video = videos[ 0 ]
292 expect(video.redundancies.files).to.have.lengthOf(4)
293 expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1)
294
295 const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists)
296
297 for (const r of redundancies) {
298 expect(r.strategy).to.be.null
299 expect(r.fileUrl).to.exist
300 expect(r.createdAt).to.exist
301 expect(r.updatedAt).to.exist
302 expect(r.expiresOn).to.be.null
303 }
304 })
305
306 it('Should manually remove a redundancy and remove it from the list', async function () {
307 this.timeout(120000)
308
309 for (const redundancyId of redundanciesToRemove) {
310 await removeVideoRedundancy({
311 url: servers[ 0 ].url,
312 accessToken: servers[ 0 ].accessToken,
313 redundancyId
314 })
315 }
316
317 {
318 const res = await listVideoRedundancies({
319 url: servers[0].url,
320 accessToken: servers[0].accessToken,
321 target: 'remote-videos',
322 sort: '-name',
323 start: 0,
324 count: 5
325 })
326
327 const videos = res.body.data
328 expect(videos).to.have.lengthOf(2)
329
330 expect(videos[ 0 ].name).to.equal('video 2 server 2')
331
332 redundanciesToRemove = []
333 const video = videos[ 0 ]
334 expect(video.redundancies.files).to.have.lengthOf(4)
335 expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1)
336
337 const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists)
338
339 for (const r of redundancies) {
340 redundanciesToRemove.push(r.id)
341 }
342 }
343 })
344
345 it('Should remove another (auto) redundancy', async function () {
346 {
347 for (const redundancyId of redundanciesToRemove) {
348 await removeVideoRedundancy({
349 url: servers[ 0 ].url,
350 accessToken: servers[ 0 ].accessToken,
351 redundancyId
352 })
353 }
354
355 const res = await listVideoRedundancies({
356 url: servers[0].url,
357 accessToken: servers[0].accessToken,
358 target: 'remote-videos',
359 sort: '-name',
360 start: 0,
361 count: 5
362 })
363
364 const videos = res.body.data
365 expect(videos[ 0 ].name).to.equal('video 1 server 2')
366 expect(videos).to.have.lengthOf(1)
367 }
368 })
369
370 after(async function () {
371 await cleanupTests(servers)
372 })
373})
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index 1cdf93aa1..f5bf130d5 100644
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -5,7 +5,8 @@ import 'mocha'
5import { VideoDetails } from '../../../../shared/models/videos' 5import { VideoDetails } from '../../../../shared/models/videos'
6import { 6import {
7 checkSegmentHash, 7 checkSegmentHash,
8 checkVideoFilesWereRemoved, cleanupTests, 8 checkVideoFilesWereRemoved,
9 cleanupTests,
9 doubleFollow, 10 doubleFollow,
10 flushAndRunMultipleServers, 11 flushAndRunMultipleServers,
11 getFollowingListPaginationAndSort, 12 getFollowingListPaginationAndSort,
@@ -28,11 +29,16 @@ import {
28import { waitJobs } from '../../../../shared/extra-utils/server/jobs' 29import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
29 30
30import * as magnetUtil from 'magnet-uri' 31import * as magnetUtil from 'magnet-uri'
31import { updateRedundancy } from '../../../../shared/extra-utils/server/redundancy' 32import {
33 addVideoRedundancy,
34 listVideoRedundancies,
35 removeVideoRedundancy,
36 updateRedundancy
37} from '../../../../shared/extra-utils/server/redundancy'
32import { ActorFollow } from '../../../../shared/models/actors' 38import { ActorFollow } from '../../../../shared/models/actors'
33import { readdir } from 'fs-extra' 39import { readdir } from 'fs-extra'
34import { join } from 'path' 40import { join } from 'path'
35import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy' 41import { VideoRedundancy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../../shared/models/redundancy'
36import { getStats } from '../../../../shared/extra-utils/server/stats' 42import { getStats } from '../../../../shared/extra-utils/server/stats'
37import { ServerStats } from '../../../../shared/models/server/server-stats.model' 43import { ServerStats } from '../../../../shared/models/server/server-stats.model'
38 44
@@ -40,6 +46,7 @@ const expect = chai.expect
40 46
41let servers: ServerInfo[] = [] 47let servers: ServerInfo[] = []
42let video1Server2UUID: string 48let video1Server2UUID: string
49let video1Server2Id: number
43 50
44function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) { 51function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
45 const parsed = magnetUtil.decode(file.magnetUri) 52 const parsed = magnetUtil.decode(file.magnetUri)
@@ -52,7 +59,19 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe
52 expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) 59 expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
53} 60}
54 61
55async function flushAndRunServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { 62async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}) {
63 const strategies: any[] = []
64
65 if (strategy !== null) {
66 strategies.push(
67 immutableAssign({
68 min_lifetime: '1 hour',
69 strategy: strategy,
70 size: '400KB'
71 }, additionalParams)
72 )
73 }
74
56 const config = { 75 const config = {
57 transcoding: { 76 transcoding: {
58 hls: { 77 hls: {
@@ -62,16 +81,11 @@ async function flushAndRunServers (strategy: VideoRedundancyStrategy, additional
62 redundancy: { 81 redundancy: {
63 videos: { 82 videos: {
64 check_interval: '5 seconds', 83 check_interval: '5 seconds',
65 strategies: [ 84 strategies
66 immutableAssign({
67 min_lifetime: '1 hour',
68 strategy: strategy,
69 size: '400KB'
70 }, additionalParams)
71 ]
72 } 85 }
73 } 86 }
74 } 87 }
88
75 servers = await flushAndRunMultipleServers(3, config) 89 servers = await flushAndRunMultipleServers(3, config)
76 90
77 // Get the access tokens 91 // Get the access tokens
@@ -80,6 +94,7 @@ async function flushAndRunServers (strategy: VideoRedundancyStrategy, additional
80 { 94 {
81 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) 95 const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
82 video1Server2UUID = res.body.video.uuid 96 video1Server2UUID = res.body.video.uuid
97 video1Server2Id = res.body.video.id
83 98
84 await viewVideo(servers[ 1 ].url, video1Server2UUID) 99 await viewVideo(servers[ 1 ].url, video1Server2UUID)
85 } 100 }
@@ -216,29 +231,38 @@ async function check1PlaylistRedundancies (videoUUID?: string) {
216 } 231 }
217} 232}
218 233
219async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { 234async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) {
235 let totalSize: number = null
236 let statsLength = 1
237
238 if (strategy !== 'manual') {
239 totalSize = 409600
240 statsLength = 2
241 }
242
220 const res = await getStats(servers[0].url) 243 const res = await getStats(servers[0].url)
221 const data: ServerStats = res.body 244 const data: ServerStats = res.body
222 245
223 expect(data.videosRedundancy).to.have.lengthOf(1) 246 expect(data.videosRedundancy).to.have.lengthOf(statsLength)
224 const stat = data.videosRedundancy[0]
225 247
248 const stat = data.videosRedundancy[0]
226 expect(stat.strategy).to.equal(strategy) 249 expect(stat.strategy).to.equal(strategy)
227 expect(stat.totalSize).to.equal(409600) 250 expect(stat.totalSize).to.equal(totalSize)
251
252 return stat
253}
254
255async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategyWithManual) {
256 const stat = await checkStatsGlobal(strategy)
257
228 expect(stat.totalUsed).to.be.at.least(1).and.below(409601) 258 expect(stat.totalUsed).to.be.at.least(1).and.below(409601)
229 expect(stat.totalVideoFiles).to.equal(4) 259 expect(stat.totalVideoFiles).to.equal(4)
230 expect(stat.totalVideos).to.equal(1) 260 expect(stat.totalVideos).to.equal(1)
231} 261}
232 262
233async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { 263async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategyWithManual) {
234 const res = await getStats(servers[0].url) 264 const stat = await checkStatsGlobal(strategy)
235 const data: ServerStats = res.body
236
237 expect(data.videosRedundancy).to.have.lengthOf(1)
238 265
239 const stat = data.videosRedundancy[0]
240 expect(stat.strategy).to.equal(strategy)
241 expect(stat.totalSize).to.equal(409600)
242 expect(stat.totalUsed).to.equal(0) 266 expect(stat.totalUsed).to.equal(0)
243 expect(stat.totalVideoFiles).to.equal(0) 267 expect(stat.totalVideoFiles).to.equal(0)
244 expect(stat.totalVideos).to.equal(0) 268 expect(stat.totalVideos).to.equal(0)
@@ -446,6 +470,74 @@ describe('Test videos redundancy', function () {
446 }) 470 })
447 }) 471 })
448 472
473 describe('With manual strategy', function () {
474 before(function () {
475 this.timeout(120000)
476
477 return flushAndRunServers(null)
478 })
479
480 it('Should have 1 webseed on the first video', async function () {
481 await check1WebSeed()
482 await check0PlaylistRedundancies()
483 await checkStatsWith1Webseed('manual')
484 })
485
486 it('Should create a redundancy on first video', async function () {
487 await addVideoRedundancy({
488 url: servers[0].url,
489 accessToken: servers[0].accessToken,
490 videoId: video1Server2Id
491 })
492 })
493
494 it('Should have 2 webseeds on the first video', async function () {
495 this.timeout(80000)
496
497 await waitJobs(servers)
498 await waitUntilLog(servers[0], 'Duplicated ', 5)
499 await waitJobs(servers)
500
501 await check2Webseeds()
502 await check1PlaylistRedundancies()
503 await checkStatsWith2Webseed('manual')
504 })
505
506 it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
507 this.timeout(80000)
508
509 const res = await listVideoRedundancies({
510 url: servers[0].url,
511 accessToken: servers[0].accessToken,
512 target: 'remote-videos'
513 })
514
515 const videos = res.body.data as VideoRedundancy[]
516 expect(videos).to.have.lengthOf(1)
517
518 const video = videos[0]
519 for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) {
520 await removeVideoRedundancy({
521 url: servers[0].url,
522 accessToken: servers[0].accessToken,
523 redundancyId: r.id
524 })
525 }
526
527 await waitJobs(servers)
528 await wait(5000)
529
530 await check1WebSeed()
531 await check0PlaylistRedundancies()
532
533 await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ])
534 })
535
536 after(async function () {
537 await cleanupTests(servers)
538 })
539 })
540
449 describe('Test expiration', function () { 541 describe('Test expiration', function () {
450 const strategy = 'recently-added' 542 const strategy = 'recently-added'
451 543
diff --git a/server/typings/models/video/video-file.ts b/server/typings/models/video/video-file.ts
index 352fe3d32..139b22b2c 100644
--- a/server/typings/models/video/video-file.ts
+++ b/server/typings/models/video/video-file.ts
@@ -1,7 +1,7 @@
1import { VideoFileModel } from '../../../models/video/video-file' 1import { VideoFileModel } from '../../../models/video/video-file'
2import { PickWith, PickWithOpt } from '../../utils' 2import { PickWith, PickWithOpt } from '../../utils'
3import { MVideo, MVideoUUID } from './video' 3import { MVideo, MVideoUUID } from './video'
4import { MVideoRedundancyFileUrl } from './video-redundancy' 4import { MVideoRedundancy, MVideoRedundancyFileUrl } from './video-redundancy'
5import { MStreamingPlaylistVideo, MStreamingPlaylist } from './video-streaming-playlist' 5import { MStreamingPlaylistVideo, MStreamingPlaylist } from './video-streaming-playlist'
6 6
7type Use<K extends keyof VideoFileModel, M> = PickWith<VideoFileModel, K, M> 7type Use<K extends keyof VideoFileModel, M> = PickWith<VideoFileModel, K, M>
@@ -22,6 +22,9 @@ export type MVideoFileStreamingPlaylistVideo = MVideoFile &
22export type MVideoFileVideoUUID = MVideoFile & 22export type MVideoFileVideoUUID = MVideoFile &
23 Use<'Video', MVideoUUID> 23 Use<'Video', MVideoUUID>
24 24
25export type MVideoFileRedundanciesAll = MVideoFile &
26 PickWithOpt<VideoFileModel, 'RedundancyVideos', MVideoRedundancy[]>
27
25export type MVideoFileRedundanciesOpt = MVideoFile & 28export type MVideoFileRedundanciesOpt = MVideoFile &
26 PickWithOpt<VideoFileModel, 'RedundancyVideos', MVideoRedundancyFileUrl[]> 29 PickWithOpt<VideoFileModel, 'RedundancyVideos', MVideoRedundancyFileUrl[]>
27 30
diff --git a/server/typings/models/video/video-streaming-playlist.ts b/server/typings/models/video/video-streaming-playlist.ts
index 436c0c072..6fd489945 100644
--- a/server/typings/models/video/video-streaming-playlist.ts
+++ b/server/typings/models/video/video-streaming-playlist.ts
@@ -1,6 +1,6 @@
1import { VideoStreamingPlaylistModel } from '../../../models/video/video-streaming-playlist' 1import { VideoStreamingPlaylistModel } from '../../../models/video/video-streaming-playlist'
2import { PickWith, PickWithOpt } from '../../utils' 2import { PickWith, PickWithOpt } from '../../utils'
3import { MVideoRedundancyFileUrl } from './video-redundancy' 3import { MVideoRedundancyFileUrl, MVideoRedundancy } from './video-redundancy'
4import { MVideo } from './video' 4import { MVideo } from './video'
5import { MVideoFile } from './video-file' 5import { MVideoFile } from './video-file'
6 6
@@ -20,6 +20,10 @@ export type MStreamingPlaylistFilesVideo = MStreamingPlaylist &
20 Use<'VideoFiles', MVideoFile[]> & 20 Use<'VideoFiles', MVideoFile[]> &
21 Use<'Video', MVideo> 21 Use<'Video', MVideo>
22 22
23export type MStreamingPlaylistRedundanciesAll = MStreamingPlaylist &
24 Use<'VideoFiles', MVideoFile[]> &
25 Use<'RedundancyVideos', MVideoRedundancy[]>
26
23export type MStreamingPlaylistRedundancies = MStreamingPlaylist & 27export type MStreamingPlaylistRedundancies = MStreamingPlaylist &
24 Use<'VideoFiles', MVideoFile[]> & 28 Use<'VideoFiles', MVideoFile[]> &
25 Use<'RedundancyVideos', MVideoRedundancyFileUrl[]> 29 Use<'RedundancyVideos', MVideoRedundancyFileUrl[]>
diff --git a/server/typings/models/video/video.ts b/server/typings/models/video/video.ts
index 7f69a91de..bcc5e5028 100644
--- a/server/typings/models/video/video.ts
+++ b/server/typings/models/video/video.ts
@@ -10,8 +10,13 @@ import {
10} from './video-channels' 10} from './video-channels'
11import { MTag } from './tag' 11import { MTag } from './tag'
12import { MVideoCaptionLanguage } from './video-caption' 12import { MVideoCaptionLanguage } from './video-caption'
13import { MStreamingPlaylistFiles, MStreamingPlaylistRedundancies, MStreamingPlaylistRedundanciesOpt } from './video-streaming-playlist' 13import {
14import { MVideoFile, MVideoFileRedundanciesOpt } from './video-file' 14 MStreamingPlaylistFiles,
15 MStreamingPlaylistRedundancies,
16 MStreamingPlaylistRedundanciesAll,
17 MStreamingPlaylistRedundanciesOpt
18} from './video-streaming-playlist'
19import { MVideoFile, MVideoFileRedundanciesAll, MVideoFileRedundanciesOpt } from './video-file'
15import { MThumbnail } from './thumbnail' 20import { MThumbnail } from './thumbnail'
16import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist' 21import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist'
17import { MScheduleVideoUpdate } from './schedule-video-update' 22import { MScheduleVideoUpdate } from './schedule-video-update'
@@ -158,6 +163,10 @@ export type MVideoForUser = MVideo &
158 Use<'VideoBlacklist', MVideoBlacklistLight> & 163 Use<'VideoBlacklist', MVideoBlacklistLight> &
159 Use<'Thumbnails', MThumbnail[]> 164 Use<'Thumbnails', MThumbnail[]>
160 165
166export type MVideoForRedundancyAPI = MVideo &
167 Use<'VideoFiles', MVideoFileRedundanciesAll[]> &
168 Use<'VideoStreamingPlaylists', MStreamingPlaylistRedundanciesAll[]>
169
161// ############################################################################ 170// ############################################################################
162 171
163// Format for API or AP object 172// Format for API or AP object
diff --git a/shared/extra-utils/server/redundancy.ts b/shared/extra-utils/server/redundancy.ts
index c39ff2c8b..7b488e23e 100644
--- a/shared/extra-utils/server/redundancy.ts
+++ b/shared/extra-utils/server/redundancy.ts
@@ -1,6 +1,7 @@
1import { makePutBodyRequest } from '../requests/requests' 1import { makeDeleteRequest, makeGetRequest, makePostBodyRequest, makePutBodyRequest } from '../requests/requests'
2import { VideoRedundanciesTarget } from '@shared/models'
2 3
3async function updateRedundancy (url: string, accessToken: string, host: string, redundancyAllowed: boolean, expectedStatus = 204) { 4function updateRedundancy (url: string, accessToken: string, host: string, redundancyAllowed: boolean, expectedStatus = 204) {
4 const path = '/api/v1/server/redundancy/' + host 5 const path = '/api/v1/server/redundancy/' + host
5 6
6 return makePutBodyRequest({ 7 return makePutBodyRequest({
@@ -12,6 +13,69 @@ async function updateRedundancy (url: string, accessToken: string, host: string,
12 }) 13 })
13} 14}
14 15
16function listVideoRedundancies (options: {
17 url: string
18 accessToken: string,
19 target: VideoRedundanciesTarget,
20 start?: number,
21 count?: number,
22 sort?: string,
23 statusCodeExpected?: number
24}) {
25 const path = '/api/v1/server/redundancy/videos'
26
27 const { url, accessToken, target, statusCodeExpected, start, count, sort } = options
28
29 return makeGetRequest({
30 url,
31 token: accessToken,
32 path,
33 query: {
34 start: start ?? 0,
35 count: count ?? 5,
36 sort: sort ?? 'name',
37 target
38 },
39 statusCodeExpected: statusCodeExpected || 200
40 })
41}
42
43function addVideoRedundancy (options: {
44 url: string,
45 accessToken: string,
46 videoId: number
47}) {
48 const path = '/api/v1/server/redundancy/videos'
49 const { url, accessToken, videoId } = options
50
51 return makePostBodyRequest({
52 url,
53 token: accessToken,
54 path,
55 fields: { videoId },
56 statusCodeExpected: 204
57 })
58}
59
60function removeVideoRedundancy (options: {
61 url: string,
62 accessToken: string,
63 redundancyId: number
64}) {
65 const { url, accessToken, redundancyId } = options
66 const path = '/api/v1/server/redundancy/videos/' + redundancyId
67
68 return makeDeleteRequest({
69 url,
70 token: accessToken,
71 path,
72 statusCodeExpected: 204
73 })
74}
75
15export { 76export {
16 updateRedundancy 77 updateRedundancy,
78 listVideoRedundancies,
79 addVideoRedundancy,
80 removeVideoRedundancy
17} 81}
diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts
index 7a77a03ad..aa13273ae 100644
--- a/shared/extra-utils/videos/videos.ts
+++ b/shared/extra-utils/videos/videos.ts
@@ -607,15 +607,28 @@ async function videoUUIDToId (url: string, id: number | string) {
607 return res.body.id 607 return res.body.id
608} 608}
609 609
610async function uploadVideoAndGetId (options: { server: ServerInfo, videoName: string, nsfw?: boolean, token?: string }) { 610async function uploadVideoAndGetId (options: {
611 server: ServerInfo,
612 videoName: string,
613 nsfw?: boolean,
614 privacy?: VideoPrivacy,
615 token?: string
616}) {
611 const videoAttrs: any = { name: options.videoName } 617 const videoAttrs: any = { name: options.videoName }
612 if (options.nsfw) videoAttrs.nsfw = options.nsfw 618 if (options.nsfw) videoAttrs.nsfw = options.nsfw
619 if (options.privacy) videoAttrs.privacy = options.privacy
613 620
614 const res = await uploadVideo(options.server.url, options.token || options.server.accessToken, videoAttrs) 621 const res = await uploadVideo(options.server.url, options.token || options.server.accessToken, videoAttrs)
615 622
616 return { id: res.body.video.id, uuid: res.body.video.uuid } 623 return { id: res.body.video.id, uuid: res.body.video.uuid }
617} 624}
618 625
626async function getLocalIdByUUID (url: string, uuid: string) {
627 const res = await getVideo(url, uuid)
628
629 return res.body.id
630}
631
619// --------------------------------------------------------------------------- 632// ---------------------------------------------------------------------------
620 633
621export { 634export {
@@ -645,5 +658,6 @@ export {
645 completeVideoCheck, 658 completeVideoCheck,
646 checkVideoFilesWereRemoved, 659 checkVideoFilesWereRemoved,
647 getPlaylistVideos, 660 getPlaylistVideos,
648 uploadVideoAndGetId 661 uploadVideoAndGetId,
662 getLocalIdByUUID
649} 663}
diff --git a/shared/models/redundancy/index.ts b/shared/models/redundancy/index.ts
index 61bf0fca7..649cc489f 100644
--- a/shared/models/redundancy/index.ts
+++ b/shared/models/redundancy/index.ts
@@ -1 +1,3 @@
1export * from './videos-redundancy.model' 1export * from './videos-redundancy-strategy.model'
2export * from './video-redundancies-filters.model'
3export * from './video-redundancy.model'
diff --git a/shared/models/redundancy/video-redundancies-filters.model.ts b/shared/models/redundancy/video-redundancies-filters.model.ts
new file mode 100644
index 000000000..05ba7dfd3
--- /dev/null
+++ b/shared/models/redundancy/video-redundancies-filters.model.ts
@@ -0,0 +1 @@
export type VideoRedundanciesTarget = 'my-videos' | 'remote-videos'
diff --git a/shared/models/redundancy/video-redundancy.model.ts b/shared/models/redundancy/video-redundancy.model.ts
new file mode 100644
index 000000000..014f69634
--- /dev/null
+++ b/shared/models/redundancy/video-redundancy.model.ts
@@ -0,0 +1,33 @@
1export interface VideoRedundancy {
2 id: number
3 name: string
4 url: string
5 uuid: string
6
7 redundancies: {
8 files: FileRedundancyInformation[]
9
10 streamingPlaylists: StreamingPlaylistRedundancyInformation[]
11 }
12}
13
14interface RedundancyInformation {
15 id: number
16 fileUrl: string
17 strategy: string
18
19 createdAt: Date | string
20 updatedAt: Date | string
21
22 expiresOn: Date | string
23
24 size: number
25}
26
27export interface FileRedundancyInformation extends RedundancyInformation {
28
29}
30
31export interface StreamingPlaylistRedundancyInformation extends RedundancyInformation {
32
33}
diff --git a/shared/models/redundancy/videos-redundancy.model.ts b/shared/models/redundancy/videos-redundancy-strategy.model.ts
index a8c2743c1..15409abf0 100644
--- a/shared/models/redundancy/videos-redundancy.model.ts
+++ b/shared/models/redundancy/videos-redundancy-strategy.model.ts
@@ -1,4 +1,5 @@
1export type VideoRedundancyStrategy = 'most-views' | 'trending' | 'recently-added' 1export type VideoRedundancyStrategy = 'most-views' | 'trending' | 'recently-added'
2export type VideoRedundancyStrategyWithManual = VideoRedundancyStrategy | 'manual'
2 3
3export type MostViewsRedundancyStrategy = { 4export type MostViewsRedundancyStrategy = {
4 strategy: 'most-views' 5 strategy: 'most-views'
@@ -19,4 +20,4 @@ export type RecentlyAddedStrategy = {
19 minLifetime: number 20 minLifetime: number
20} 21}
21 22
22export type VideosRedundancy = MostViewsRedundancyStrategy | TrendingRedundancyStrategy | RecentlyAddedStrategy 23export type VideosRedundancyStrategy = MostViewsRedundancyStrategy | TrendingRedundancyStrategy | RecentlyAddedStrategy
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts
index b82a633b2..19fd4c659 100644
--- a/shared/models/server/job.model.ts
+++ b/shared/models/server/job.model.ts
@@ -9,7 +9,8 @@ export type JobType = 'activitypub-http-unicast' |
9 'email' | 9 'email' |
10 'video-import' | 10 'video-import' |
11 'videos-views' | 11 'videos-views' |
12 'activitypub-refresher' 12 'activitypub-refresher' |
13 'video-redundancy'
13 14
14export interface Job { 15export interface Job {
15 id: number 16 id: number
diff --git a/shared/models/server/server-stats.model.ts b/shared/models/server/server-stats.model.ts
index 74f3de5d3..11778e6ed 100644
--- a/shared/models/server/server-stats.model.ts
+++ b/shared/models/server/server-stats.model.ts
@@ -1,4 +1,4 @@
1import { VideoRedundancyStrategy } from '../redundancy' 1import { VideoRedundancyStrategyWithManual } from '../redundancy'
2 2
3export interface ServerStats { 3export interface ServerStats {
4 totalUsers: number 4 totalUsers: number
@@ -13,11 +13,13 @@ export interface ServerStats {
13 totalInstanceFollowers: number 13 totalInstanceFollowers: number
14 totalInstanceFollowing: number 14 totalInstanceFollowing: number
15 15
16 videosRedundancy: { 16 videosRedundancy: VideosRedundancyStats[]
17 strategy: VideoRedundancyStrategy 17}
18 totalSize: number 18
19 totalUsed: number 19export interface VideosRedundancyStats {
20 totalVideoFiles: number 20 strategy: VideoRedundancyStrategyWithManual
21 totalVideos: number 21 totalSize: number
22 }[] 22 totalUsed: number
23 totalVideoFiles: number
24 totalVideos: number
23} 25}
diff --git a/shared/models/users/user-right.enum.ts b/shared/models/users/user-right.enum.ts
index 4a28a229d..2f88a65de 100644
--- a/shared/models/users/user-right.enum.ts
+++ b/shared/models/users/user-right.enum.ts
@@ -33,5 +33,7 @@ export enum UserRight {
33 SEE_ALL_VIDEOS, 33 SEE_ALL_VIDEOS,
34 CHANGE_VIDEO_OWNERSHIP, 34 CHANGE_VIDEO_OWNERSHIP,
35 35
36 MANAGE_PLUGINS 36 MANAGE_PLUGINS,
37
38 MANAGE_VIDEOS_REDUNDANCIES
37} 39}
diff --git a/shared/models/videos/video.model.ts b/shared/models/videos/video.model.ts
index 7576439fe..a69152759 100644
--- a/shared/models/videos/video.model.ts
+++ b/shared/models/videos/video.model.ts
@@ -1,4 +1,4 @@
1import { AccountSummary, VideoChannelSummary, VideoResolution, VideoState } from '../../index' 1import { AccountSummary, VideoChannelSummary, VideoState } from '../../index'
2import { Account } from '../actors' 2import { Account } from '../actors'
3import { VideoChannel } from './channel/video-channel.model' 3import { VideoChannel } from './channel/video-channel.model'
4import { VideoPrivacy } from './video-privacy.enum' 4import { VideoPrivacy } from './video-privacy.enum'