diff options
author | Chocobozzz <me@florianbigard.com> | 2020-01-10 10:11:28 +0100 |
---|---|---|
committer | Chocobozzz <chocobozzz@cpy.re> | 2020-01-28 11:35:26 +0100 |
commit | b764380ac23f4e9d4677d08acdc3474c2931a16d (patch) | |
tree | 0d2c440ed8b56c35e47f2274586a11da63852086 | |
parent | 3ae0bbd23c6f1b2790975328d8eae6a8317c223d (diff) | |
download | PeerTube-b764380ac23f4e9d4677d08acdc3474c2931a16d.tar.gz PeerTube-b764380ac23f4e9d4677d08acdc3474c2931a16d.tar.zst PeerTube-b764380ac23f4e9d4677d08acdc3474c2931a16d.zip |
Add ability to list redundancies
64 files changed, 1806 insertions, 194 deletions
diff --git a/client/package.json b/client/package.json index cd0a82aa4..7205dbe8f 100644 --- a/client/package.json +++ b/client/package.json | |||
@@ -77,6 +77,7 @@ | |||
77 | "bootstrap": "^4.1.3", | 77 | "bootstrap": "^4.1.3", |
78 | "buffer": "^5.1.0", | 78 | "buffer": "^5.1.0", |
79 | "cache-chunk-store": "^3.0.0", | 79 | "cache-chunk-store": "^3.0.0", |
80 | "chart.js": "^2.9.3", | ||
80 | "codelyzer": "^5.0.1", | 81 | "codelyzer": "^5.0.1", |
81 | "core-js": "^3.1.4", | 82 | "core-js": "^3.1.4", |
82 | "css-loader": "^3.1.0", | 83 | "css-loader": "^3.1.0", |
diff --git a/client/src/app/+admin/admin.component.html b/client/src/app/+admin/admin.component.html index 9a3d90c18..0d06aaedc 100644 --- a/client/src/app/+admin/admin.component.html +++ b/client/src/app/+admin/admin.component.html | |||
@@ -5,7 +5,7 @@ | |||
5 | </a> | 5 | </a> |
6 | 6 | ||
7 | <a i18n *ngIf="hasServerFollowRight()" routerLink="/admin/follows" routerLinkActive="active" class="title-page"> | 7 | <a i18n *ngIf="hasServerFollowRight()" routerLink="/admin/follows" routerLinkActive="active" class="title-page"> |
8 | Manage follows | 8 | Follows & redundancies |
9 | </a> | 9 | </a> |
10 | 10 | ||
11 | <a i18n *ngIf="hasVideoAbusesRight() || hasVideoBlacklistRight()" routerLink="/admin/moderation" routerLinkActive="active" class="title-page"> | 11 | <a i18n *ngIf="hasVideoAbusesRight() || hasVideoBlacklistRight()" routerLink="/admin/moderation" routerLinkActive="active" class="title-page"> |
diff --git a/client/src/app/+admin/admin.module.ts b/client/src/app/+admin/admin.module.ts index 9c56b5750..fdbe70314 100644 --- a/client/src/app/+admin/admin.module.ts +++ b/client/src/app/+admin/admin.module.ts | |||
@@ -5,7 +5,7 @@ import { TableModule } from 'primeng/table' | |||
5 | import { SharedModule } from '../shared' | 5 | import { SharedModule } from '../shared' |
6 | import { AdminRoutingModule } from './admin-routing.module' | 6 | import { AdminRoutingModule } from './admin-routing.module' |
7 | import { AdminComponent } from './admin.component' | 7 | import { AdminComponent } from './admin.component' |
8 | import { FollowersListComponent, FollowingAddComponent, FollowsComponent } from './follows' | 8 | import { FollowersListComponent, FollowingAddComponent, FollowsComponent, VideoRedundanciesListComponent } from './follows' |
9 | import { FollowingListComponent } from './follows/following-list/following-list.component' | 9 | import { FollowingListComponent } from './follows/following-list/following-list.component' |
10 | import { UserCreateComponent, UserListComponent, UserPasswordComponent, UsersComponent, UserUpdateComponent } from './users' | 10 | import { UserCreateComponent, UserListComponent, UserPasswordComponent, UsersComponent, UserUpdateComponent } from './users' |
11 | import { | 11 | import { |
@@ -16,7 +16,6 @@ import { | |||
16 | } from './moderation' | 16 | } from './moderation' |
17 | import { ModerationComponent } from '@app/+admin/moderation/moderation.component' | 17 | import { ModerationComponent } from '@app/+admin/moderation/moderation.component' |
18 | import { RedundancyCheckboxComponent } from '@app/+admin/follows/shared/redundancy-checkbox.component' | 18 | import { RedundancyCheckboxComponent } from '@app/+admin/follows/shared/redundancy-checkbox.component' |
19 | import { RedundancyService } from '@app/+admin/follows/shared/redundancy.service' | ||
20 | import { InstanceAccountBlocklistComponent, InstanceServerBlocklistComponent } from '@app/+admin/moderation/instance-blocklist' | 19 | import { InstanceAccountBlocklistComponent, InstanceServerBlocklistComponent } from '@app/+admin/moderation/instance-blocklist' |
21 | import { JobsComponent } from '@app/+admin/system/jobs/jobs.component' | 20 | import { JobsComponent } from '@app/+admin/system/jobs/jobs.component' |
22 | import { JobService, LogsComponent, LogsService, SystemComponent } from '@app/+admin/system' | 21 | import { JobService, LogsComponent, LogsService, SystemComponent } from '@app/+admin/system' |
@@ -27,13 +26,18 @@ import { PluginSearchComponent } from '@app/+admin/plugins/plugin-search/plugin- | |||
27 | import { PluginShowInstalledComponent } from '@app/+admin/plugins/plugin-show-installed/plugin-show-installed.component' | 26 | import { PluginShowInstalledComponent } from '@app/+admin/plugins/plugin-show-installed/plugin-show-installed.component' |
28 | import { SelectButtonModule } from 'primeng/selectbutton' | 27 | import { SelectButtonModule } from 'primeng/selectbutton' |
29 | import { PluginApiService } from '@app/+admin/plugins/shared/plugin-api.service' | 28 | import { PluginApiService } from '@app/+admin/plugins/shared/plugin-api.service' |
29 | import { VideoRedundancyInformationComponent } from '@app/+admin/follows/video-redundancies-list/video-redundancy-information.component' | ||
30 | import { ChartModule } from 'primeng/chart' | ||
30 | 31 | ||
31 | @NgModule({ | 32 | @NgModule({ |
32 | imports: [ | 33 | imports: [ |
33 | AdminRoutingModule, | 34 | AdminRoutingModule, |
35 | |||
36 | SharedModule, | ||
37 | |||
34 | TableModule, | 38 | TableModule, |
35 | SelectButtonModule, | 39 | SelectButtonModule, |
36 | SharedModule | 40 | ChartModule |
37 | ], | 41 | ], |
38 | 42 | ||
39 | declarations: [ | 43 | declarations: [ |
@@ -44,6 +48,8 @@ import { PluginApiService } from '@app/+admin/plugins/shared/plugin-api.service' | |||
44 | FollowersListComponent, | 48 | FollowersListComponent, |
45 | FollowingListComponent, | 49 | FollowingListComponent, |
46 | RedundancyCheckboxComponent, | 50 | RedundancyCheckboxComponent, |
51 | VideoRedundanciesListComponent, | ||
52 | VideoRedundancyInformationComponent, | ||
47 | 53 | ||
48 | UsersComponent, | 54 | UsersComponent, |
49 | UserCreateComponent, | 55 | UserCreateComponent, |
@@ -78,7 +84,6 @@ import { PluginApiService } from '@app/+admin/plugins/shared/plugin-api.service' | |||
78 | ], | 84 | ], |
79 | 85 | ||
80 | providers: [ | 86 | providers: [ |
81 | RedundancyService, | ||
82 | JobService, | 87 | JobService, |
83 | LogsService, | 88 | LogsService, |
84 | DebugService, | 89 | DebugService, |
diff --git a/client/src/app/+admin/follows/follows.component.html b/client/src/app/+admin/follows/follows.component.html index 21d477132..46581daf9 100644 --- a/client/src/app/+admin/follows/follows.component.html +++ b/client/src/app/+admin/follows/follows.component.html | |||
@@ -1,5 +1,5 @@ | |||
1 | <div class="admin-sub-header"> | 1 | <div class="admin-sub-header"> |
2 | <div i18n class="form-sub-title">Manage follows</div> | 2 | <div i18n class="form-sub-title">Follows & redundancies</div> |
3 | 3 | ||
4 | <div class="admin-sub-nav"> | 4 | <div class="admin-sub-nav"> |
5 | <a i18n routerLink="following-list" routerLinkActive="active">Following</a> | 5 | <a i18n routerLink="following-list" routerLinkActive="active">Following</a> |
@@ -7,7 +7,9 @@ | |||
7 | <a i18n routerLink="following-add" routerLinkActive="active">Follow</a> | 7 | <a i18n routerLink="following-add" routerLinkActive="active">Follow</a> |
8 | 8 | ||
9 | <a i18n routerLink="followers-list" routerLinkActive="active">Followers</a> | 9 | <a i18n routerLink="followers-list" routerLinkActive="active">Followers</a> |
10 | |||
11 | <a i18n routerLink="video-redundancies-list" routerLinkActive="active">Video redundancies</a> | ||
10 | </div> | 12 | </div> |
11 | </div> | 13 | </div> |
12 | 14 | ||
13 | <router-outlet></router-outlet> \ No newline at end of file | 15 | <router-outlet></router-outlet> |
diff --git a/client/src/app/+admin/follows/follows.routes.ts b/client/src/app/+admin/follows/follows.routes.ts index e84c79e82..298733eb0 100644 --- a/client/src/app/+admin/follows/follows.routes.ts +++ b/client/src/app/+admin/follows/follows.routes.ts | |||
@@ -6,6 +6,7 @@ import { FollowingAddComponent } from './following-add' | |||
6 | import { FollowersListComponent } from './followers-list' | 6 | import { FollowersListComponent } from './followers-list' |
7 | import { UserRight } from '../../../../../shared' | 7 | import { UserRight } from '../../../../../shared' |
8 | import { FollowingListComponent } from './following-list/following-list.component' | 8 | import { FollowingListComponent } from './following-list/following-list.component' |
9 | import { VideoRedundanciesListComponent } from '@app/+admin/follows/video-redundancies-list' | ||
9 | 10 | ||
10 | export const FollowsRoutes: Routes = [ | 11 | export const FollowsRoutes: Routes = [ |
11 | { | 12 | { |
@@ -47,6 +48,10 @@ export const FollowsRoutes: Routes = [ | |||
47 | title: 'Add follow' | 48 | title: 'Add follow' |
48 | } | 49 | } |
49 | } | 50 | } |
51 | }, | ||
52 | { | ||
53 | path: 'video-redundancies-list', | ||
54 | component: VideoRedundanciesListComponent | ||
50 | } | 55 | } |
51 | ] | 56 | ] |
52 | } | 57 | } |
diff --git a/client/src/app/+admin/follows/index.ts b/client/src/app/+admin/follows/index.ts index e94f33710..4fcb35cb1 100644 --- a/client/src/app/+admin/follows/index.ts +++ b/client/src/app/+admin/follows/index.ts | |||
@@ -1,5 +1,6 @@ | |||
1 | export * from './following-add' | 1 | export * from './following-add' |
2 | export * from './followers-list' | 2 | export * from './followers-list' |
3 | export * from './following-list' | 3 | export * from './following-list' |
4 | export * from './video-redundancies-list' | ||
4 | export * from './follows.component' | 5 | export * from './follows.component' |
5 | export * from './follows.routes' | 6 | export * from './follows.routes' |
diff --git a/client/src/app/+admin/follows/shared/redundancy-checkbox.component.ts b/client/src/app/+admin/follows/shared/redundancy-checkbox.component.ts index fa1da26bf..9d7883d97 100644 --- a/client/src/app/+admin/follows/shared/redundancy-checkbox.component.ts +++ b/client/src/app/+admin/follows/shared/redundancy-checkbox.component.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { Component, Input } from '@angular/core' | 1 | import { Component, Input } from '@angular/core' |
2 | import { Notifier } from '@app/core' | 2 | import { Notifier } from '@app/core' |
3 | import { I18n } from '@ngx-translate/i18n-polyfill' | 3 | import { I18n } from '@ngx-translate/i18n-polyfill' |
4 | import { RedundancyService } from '@app/+admin/follows/shared/redundancy.service' | 4 | import { RedundancyService } from '@app/shared/video/redundancy.service' |
5 | 5 | ||
6 | @Component({ | 6 | @Component({ |
7 | selector: 'my-redundancy-checkbox', | 7 | selector: 'my-redundancy-checkbox', |
diff --git a/client/src/app/+admin/follows/shared/redundancy.service.ts b/client/src/app/+admin/follows/shared/redundancy.service.ts deleted file mode 100644 index 87ae01c04..000000000 --- a/client/src/app/+admin/follows/shared/redundancy.service.ts +++ /dev/null | |||
@@ -1,28 +0,0 @@ | |||
1 | import { catchError, map } from 'rxjs/operators' | ||
2 | import { HttpClient } from '@angular/common/http' | ||
3 | import { Injectable } from '@angular/core' | ||
4 | import { RestExtractor } from '@app/shared' | ||
5 | import { environment } from '../../../../environments/environment' | ||
6 | |||
7 | @Injectable() | ||
8 | export class RedundancyService { | ||
9 | static BASE_USER_SUBSCRIPTIONS_URL = environment.apiUrl + '/api/v1/server/redundancy' | ||
10 | |||
11 | constructor ( | ||
12 | private authHttp: HttpClient, | ||
13 | private restExtractor: RestExtractor | ||
14 | ) { } | ||
15 | |||
16 | updateRedundancy (host: string, redundancyAllowed: boolean) { | ||
17 | const url = RedundancyService.BASE_USER_SUBSCRIPTIONS_URL + '/' + host | ||
18 | |||
19 | const body = { redundancyAllowed } | ||
20 | |||
21 | return this.authHttp.put(url, body) | ||
22 | .pipe( | ||
23 | map(this.restExtractor.extractDataBool), | ||
24 | catchError(err => this.restExtractor.handleError(err)) | ||
25 | ) | ||
26 | } | ||
27 | |||
28 | } | ||
diff --git a/client/src/app/+admin/follows/video-redundancies-list/index.ts b/client/src/app/+admin/follows/video-redundancies-list/index.ts new file mode 100644 index 000000000..6a7c7f483 --- /dev/null +++ b/client/src/app/+admin/follows/video-redundancies-list/index.ts | |||
@@ -0,0 +1 @@ | |||
export * from './video-redundancies-list.component' | |||
diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.html b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.html new file mode 100644 index 000000000..80c66ec60 --- /dev/null +++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.html | |||
@@ -0,0 +1,82 @@ | |||
1 | <div class="admin-sub-header"> | ||
2 | <div i18n class="form-sub-title">Video redundancies list</div> | ||
3 | |||
4 | <div class="select-filter-block"> | ||
5 | <label for="displayType" i18n>Display</label> | ||
6 | |||
7 | <div class="peertube-select-container"> | ||
8 | <select id="displayType" name="displayType" [(ngModel)]="displayType" (ngModelChange)="onDisplayTypeChanged()"> | ||
9 | <option value="my-videos">My videos duplicated by remote instances</option> | ||
10 | <option value="remote-videos">Remote videos duplicated by my instance</option> | ||
11 | </select> | ||
12 | </div> | ||
13 | </div> | ||
14 | </div> | ||
15 | |||
16 | <p-table | ||
17 | [value]="videoRedundancies" [lazy]="true" [paginator]="true" [totalRecords]="totalRecords" [rows]="rowsPerPage" | ||
18 | [sortField]="sort.field" [sortOrder]="sort.order" (onLazyLoad)="loadLazy($event)" dataKey="id" | ||
19 | > | ||
20 | <ng-template pTemplate="header"> | ||
21 | <tr> | ||
22 | <th i18n *ngIf="isDisplayingRemoteVideos()">Strategy</th> | ||
23 | <th i18n pSortableColumn="name">Video name <p-sortIcon field="name"></p-sortIcon></th> | ||
24 | <th i18n>Video URL</th> | ||
25 | <th i18n *ngIf="isDisplayingRemoteVideos()">Total size</th> | ||
26 | <th></th> | ||
27 | </tr> | ||
28 | </ng-template> | ||
29 | |||
30 | <ng-template pTemplate="body" let-redundancy> | ||
31 | <tr class="expander" [pRowToggler]="redundancy"> | ||
32 | <td *ngIf="isDisplayingRemoteVideos()">{{ getRedundancyStrategy(redundancy) }}</td> | ||
33 | |||
34 | <td>{{ redundancy.name }}</td> | ||
35 | |||
36 | <td> | ||
37 | <a target="_blank" rel="noopener noreferrer" [href]="redundancy.url">{{ redundancy.url }}</a> | ||
38 | </td> | ||
39 | |||
40 | <td *ngIf="isDisplayingRemoteVideos()">{{ getTotalSize(redundancy) | bytes: 1 }}</td> | ||
41 | |||
42 | <td class="action-cell"> | ||
43 | <my-delete-button (click)="removeRedundancy(redundancy)"></my-delete-button> | ||
44 | </td> | ||
45 | </tr> | ||
46 | </ng-template> | ||
47 | |||
48 | <ng-template pTemplate="rowexpansion" let-redundancy> | ||
49 | <tr> | ||
50 | <td colspan="2"> | ||
51 | <div *ngFor="let file of redundancy.redundancies.files" class="expansion-block"> | ||
52 | <my-video-redundancy-information [redundancyElement]="file"></my-video-redundancy-information> | ||
53 | </div> | ||
54 | </td> | ||
55 | </tr> | ||
56 | |||
57 | <tr> | ||
58 | <td colspan="2"> | ||
59 | <div *ngFor="let playlist of redundancy.redundancies.streamingPlaylists"> | ||
60 | <my-video-redundancy-information [redundancyElement]="playlist"></my-video-redundancy-information> | ||
61 | </div> | ||
62 | </td> | ||
63 | </tr> | ||
64 | </ng-template> | ||
65 | </p-table> | ||
66 | |||
67 | |||
68 | <div class="redundancies-charts" *ngIf="isDisplayingRemoteVideos()"> | ||
69 | <div class="form-sub-title" i18n>Enabled strategies stats</div> | ||
70 | |||
71 | <div class="chart-blocks"> | ||
72 | |||
73 | <div *ngIf="noRedundancies" i18n class="no-results"> | ||
74 | No redundancy strategy is enabled on your instance. | ||
75 | </div> | ||
76 | |||
77 | <div class="chart-block" *ngFor="let r of redundanciesGraphsData"> | ||
78 | <p-chart type="pie" [data]="r.graphData" [options]="r.options" width="300px" height="300px"></p-chart> | ||
79 | </div> | ||
80 | |||
81 | </div> | ||
82 | </div> | ||
diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.scss b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.scss new file mode 100644 index 000000000..05018c281 --- /dev/null +++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.scss | |||
@@ -0,0 +1,37 @@ | |||
1 | @import '_variables'; | ||
2 | @import '_mixins'; | ||
3 | |||
4 | .expansion-block { | ||
5 | margin-bottom: 20px; | ||
6 | } | ||
7 | |||
8 | .admin-sub-header { | ||
9 | align-items: flex-end; | ||
10 | |||
11 | .select-filter-block { | ||
12 | &:not(:last-child) { | ||
13 | margin-right: 10px; | ||
14 | } | ||
15 | |||
16 | label { | ||
17 | margin-bottom: 2px; | ||
18 | } | ||
19 | |||
20 | .peertube-select-container { | ||
21 | @include peertube-select-container(auto); | ||
22 | } | ||
23 | } | ||
24 | } | ||
25 | |||
26 | .redundancies-charts { | ||
27 | margin-top: 50px; | ||
28 | |||
29 | .chart-blocks { | ||
30 | display: flex; | ||
31 | justify-content: center; | ||
32 | |||
33 | .chart-block { | ||
34 | margin: 0 20px; | ||
35 | } | ||
36 | } | ||
37 | } | ||
diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts new file mode 100644 index 000000000..4b41d1d86 --- /dev/null +++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts | |||
@@ -0,0 +1,178 @@ | |||
1 | import { Component, OnInit } from '@angular/core' | ||
2 | import { Notifier, ServerService } from '@app/core' | ||
3 | import { SortMeta } from 'primeng/api' | ||
4 | import { ConfirmService } from '../../../core/confirm/confirm.service' | ||
5 | import { RestPagination, RestTable } from '../../../shared' | ||
6 | import { I18n } from '@ngx-translate/i18n-polyfill' | ||
7 | import { VideoRedundanciesTarget, VideoRedundancy } from '@shared/models' | ||
8 | import { peertubeLocalStorage } from '@app/shared/misc/peertube-web-storage' | ||
9 | import { VideosRedundancyStats } from '@shared/models/server' | ||
10 | import { BytesPipe } from 'ngx-pipes' | ||
11 | import { RedundancyService } from '@app/shared/video/redundancy.service' | ||
12 | |||
13 | @Component({ | ||
14 | selector: 'my-video-redundancies-list', | ||
15 | templateUrl: './video-redundancies-list.component.html', | ||
16 | styleUrls: [ './video-redundancies-list.component.scss' ] | ||
17 | }) | ||
18 | export class VideoRedundanciesListComponent extends RestTable implements OnInit { | ||
19 | private static LOCAL_STORAGE_DISPLAY_TYPE = 'video-redundancies-list-display-type' | ||
20 | |||
21 | videoRedundancies: VideoRedundancy[] = [] | ||
22 | totalRecords = 0 | ||
23 | rowsPerPage = 10 | ||
24 | |||
25 | sort: SortMeta = { field: 'name', order: 1 } | ||
26 | pagination: RestPagination = { count: this.rowsPerPage, start: 0 } | ||
27 | displayType: VideoRedundanciesTarget = 'my-videos' | ||
28 | |||
29 | redundanciesGraphsData: { stats: VideosRedundancyStats, graphData: object, options: object }[] = [] | ||
30 | |||
31 | noRedundancies = false | ||
32 | |||
33 | private bytesPipe: BytesPipe | ||
34 | |||
35 | constructor ( | ||
36 | private notifier: Notifier, | ||
37 | private confirmService: ConfirmService, | ||
38 | private redundancyService: RedundancyService, | ||
39 | private serverService: ServerService, | ||
40 | private i18n: I18n | ||
41 | ) { | ||
42 | super() | ||
43 | |||
44 | this.bytesPipe = new BytesPipe() | ||
45 | } | ||
46 | |||
47 | ngOnInit () { | ||
48 | this.loadSelectLocalStorage() | ||
49 | |||
50 | this.initialize() | ||
51 | |||
52 | this.serverService.getServerStats() | ||
53 | .subscribe(res => { | ||
54 | const redundancies = res.videosRedundancy | ||
55 | |||
56 | if (redundancies.length === 0) this.noRedundancies = true | ||
57 | |||
58 | for (const r of redundancies) { | ||
59 | this.buildPieData(r) | ||
60 | } | ||
61 | }) | ||
62 | } | ||
63 | |||
64 | isDisplayingRemoteVideos () { | ||
65 | return this.displayType === 'remote-videos' | ||
66 | } | ||
67 | |||
68 | getTotalSize (redundancy: VideoRedundancy) { | ||
69 | return redundancy.redundancies.files.reduce((a, b) => a + b.size, 0) + | ||
70 | redundancy.redundancies.streamingPlaylists.reduce((a, b) => a + b.size, 0) | ||
71 | } | ||
72 | |||
73 | onDisplayTypeChanged () { | ||
74 | this.pagination.start = 0 | ||
75 | this.saveSelectLocalStorage() | ||
76 | |||
77 | this.loadData() | ||
78 | } | ||
79 | |||
80 | getRedundancyStrategy (redundancy: VideoRedundancy) { | ||
81 | if (redundancy.redundancies.files.length !== 0) return redundancy.redundancies.files[0].strategy | ||
82 | if (redundancy.redundancies.streamingPlaylists.length !== 0) return redundancy.redundancies.streamingPlaylists[0].strategy | ||
83 | |||
84 | return '' | ||
85 | } | ||
86 | |||
87 | buildPieData (stats: VideosRedundancyStats) { | ||
88 | const totalSize = stats.totalSize | ||
89 | ? stats.totalSize - stats.totalUsed | ||
90 | : stats.totalUsed | ||
91 | |||
92 | if (totalSize === 0) return | ||
93 | |||
94 | this.redundanciesGraphsData.push({ | ||
95 | stats, | ||
96 | graphData: { | ||
97 | labels: [ this.i18n('Used'), this.i18n('Available') ], | ||
98 | datasets: [ | ||
99 | { | ||
100 | data: [ stats.totalUsed, totalSize ], | ||
101 | backgroundColor: [ | ||
102 | '#FF6384', | ||
103 | '#36A2EB' | ||
104 | ], | ||
105 | hoverBackgroundColor: [ | ||
106 | '#FF6384', | ||
107 | '#36A2EB' | ||
108 | ] | ||
109 | } | ||
110 | ] | ||
111 | }, | ||
112 | options: { | ||
113 | title: { | ||
114 | display: true, | ||
115 | text: stats.strategy | ||
116 | }, | ||
117 | |||
118 | tooltips: { | ||
119 | callbacks: { | ||
120 | label: (tooltipItem: any, data: any) => { | ||
121 | const dataset = data.datasets[tooltipItem.datasetIndex] | ||
122 | let label = data.labels[tooltipItem.index] | ||
123 | if (label) label += ': ' | ||
124 | else label = '' | ||
125 | |||
126 | label += this.bytesPipe.transform(dataset.data[tooltipItem.index], 1) | ||
127 | return label | ||
128 | } | ||
129 | } | ||
130 | } | ||
131 | } | ||
132 | }) | ||
133 | } | ||
134 | |||
135 | async removeRedundancy (redundancy: VideoRedundancy) { | ||
136 | const message = this.i18n('Do you really want to remove this video redundancy?') | ||
137 | const res = await this.confirmService.confirm(message, this.i18n('Remove redundancy')) | ||
138 | if (res === false) return | ||
139 | |||
140 | this.redundancyService.removeVideoRedundancies(redundancy) | ||
141 | .subscribe( | ||
142 | () => { | ||
143 | this.notifier.success(this.i18n('Video redundancies removed!')) | ||
144 | this.loadData() | ||
145 | }, | ||
146 | |||
147 | err => this.notifier.error(err.message) | ||
148 | ) | ||
149 | |||
150 | } | ||
151 | |||
152 | protected loadData () { | ||
153 | const options = { | ||
154 | pagination: this.pagination, | ||
155 | sort: this.sort, | ||
156 | target: this.displayType | ||
157 | } | ||
158 | |||
159 | this.redundancyService.listVideoRedundancies(options) | ||
160 | .subscribe( | ||
161 | resultList => { | ||
162 | this.videoRedundancies = resultList.data | ||
163 | this.totalRecords = resultList.total | ||
164 | }, | ||
165 | |||
166 | err => this.notifier.error(err.message) | ||
167 | ) | ||
168 | } | ||
169 | |||
170 | private loadSelectLocalStorage () { | ||
171 | const displayType = peertubeLocalStorage.getItem(VideoRedundanciesListComponent.LOCAL_STORAGE_DISPLAY_TYPE) | ||
172 | if (displayType) this.displayType = displayType as VideoRedundanciesTarget | ||
173 | } | ||
174 | |||
175 | private saveSelectLocalStorage () { | ||
176 | peertubeLocalStorage.setItem(VideoRedundanciesListComponent.LOCAL_STORAGE_DISPLAY_TYPE, this.displayType) | ||
177 | } | ||
178 | } | ||
diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.html b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.html new file mode 100644 index 000000000..a379520e3 --- /dev/null +++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.html | |||
@@ -0,0 +1,24 @@ | |||
1 | <div> | ||
2 | <span class="label">Url</span> | ||
3 | <a target="_blank" rel="noopener noreferrer" [href]="redundancyElement.fileUrl">{{ redundancyElement.fileUrl }}</a> | ||
4 | </div> | ||
5 | |||
6 | <div> | ||
7 | <span class="label">Created on</span> | ||
8 | <span>{{ redundancyElement.createdAt | date: 'medium' }}</span> | ||
9 | </div> | ||
10 | |||
11 | <div> | ||
12 | <span class="label">Expires on</span> | ||
13 | <span>{{ redundancyElement.expiresOn | date: 'medium' }}</span> | ||
14 | </div> | ||
15 | |||
16 | <div> | ||
17 | <span class="label">Size</span> | ||
18 | <span>{{ redundancyElement.size | bytes: 1 }}</span> | ||
19 | </div> | ||
20 | |||
21 | <div *ngIf="redundancyElement.strategy"> | ||
22 | <span class="label">Strategy</span> | ||
23 | <span>{{ redundancyElement.strategy }}</span> | ||
24 | </div> | ||
diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.scss b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.scss new file mode 100644 index 000000000..6b09fbb01 --- /dev/null +++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.scss | |||
@@ -0,0 +1,8 @@ | |||
1 | @import '_variables'; | ||
2 | @import '_mixins'; | ||
3 | |||
4 | .label { | ||
5 | display: inline-block; | ||
6 | min-width: 100px; | ||
7 | font-weight: $font-semibold; | ||
8 | } | ||
diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts new file mode 100644 index 000000000..6f3090c08 --- /dev/null +++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts | |||
@@ -0,0 +1,11 @@ | |||
1 | import { Component, Input } from '@angular/core' | ||
2 | import { FileRedundancyInformation, StreamingPlaylistRedundancyInformation } from '@shared/models' | ||
3 | |||
4 | @Component({ | ||
5 | selector: 'my-video-redundancy-information', | ||
6 | templateUrl: './video-redundancy-information.component.html', | ||
7 | styleUrls: [ './video-redundancy-information.component.scss' ] | ||
8 | }) | ||
9 | export class VideoRedundancyInformationComponent { | ||
10 | @Input() redundancyElement: FileRedundancyInformation | StreamingPlaylistRedundancyInformation | ||
11 | } | ||
diff --git a/client/src/app/+admin/system/jobs/jobs.component.ts b/client/src/app/+admin/system/jobs/jobs.component.ts index 20c8ea71a..bc40452cf 100644 --- a/client/src/app/+admin/system/jobs/jobs.component.ts +++ b/client/src/app/+admin/system/jobs/jobs.component.ts | |||
@@ -16,8 +16,8 @@ import { JobTypeClient } from '../../../../types/job-type-client.type' | |||
16 | styleUrls: [ './jobs.component.scss' ] | 16 | styleUrls: [ './jobs.component.scss' ] |
17 | }) | 17 | }) |
18 | export class JobsComponent extends RestTable implements OnInit { | 18 | export class JobsComponent extends RestTable implements OnInit { |
19 | private static JOB_STATE_LOCAL_STORAGE_STATE = 'jobs-list-state' | 19 | private static LOCAL_STORAGE_STATE = 'jobs-list-state' |
20 | private static JOB_STATE_LOCAL_STORAGE_TYPE = 'jobs-list-type' | 20 | private static LOCAL_STORAGE_TYPE = 'jobs-list-type' |
21 | 21 | ||
22 | jobState: JobStateClient = 'waiting' | 22 | jobState: JobStateClient = 'waiting' |
23 | jobStates: JobStateClient[] = [ 'active', 'completed', 'failed', 'waiting', 'delayed' ] | 23 | jobStates: JobStateClient[] = [ 'active', 'completed', 'failed', 'waiting', 'delayed' ] |
@@ -34,7 +34,8 @@ export class JobsComponent extends RestTable implements OnInit { | |||
34 | 'video-file-import', | 34 | 'video-file-import', |
35 | 'video-import', | 35 | 'video-import', |
36 | 'videos-views', | 36 | 'videos-views', |
37 | 'activitypub-refresher' | 37 | 'activitypub-refresher', |
38 | 'video-redundancy' | ||
38 | ] | 39 | ] |
39 | 40 | ||
40 | jobs: Job[] = [] | 41 | jobs: Job[] = [] |
@@ -77,15 +78,15 @@ export class JobsComponent extends RestTable implements OnInit { | |||
77 | } | 78 | } |
78 | 79 | ||
79 | private loadJobStateAndType () { | 80 | private loadJobStateAndType () { |
80 | const state = peertubeLocalStorage.getItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_STATE) | 81 | const state = peertubeLocalStorage.getItem(JobsComponent.LOCAL_STORAGE_STATE) |
81 | if (state) this.jobState = state as JobState | 82 | if (state) this.jobState = state as JobState |
82 | 83 | ||
83 | const type = peertubeLocalStorage.getItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_TYPE) | 84 | const type = peertubeLocalStorage.getItem(JobsComponent.LOCAL_STORAGE_TYPE) |
84 | if (type) this.jobType = type as JobType | 85 | if (type) this.jobType = type as JobType |
85 | } | 86 | } |
86 | 87 | ||
87 | private saveJobStateAndType () { | 88 | private saveJobStateAndType () { |
88 | peertubeLocalStorage.setItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_STATE, this.jobState) | 89 | peertubeLocalStorage.setItem(JobsComponent.LOCAL_STORAGE_STATE, this.jobState) |
89 | peertubeLocalStorage.setItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_TYPE, this.jobType) | 90 | peertubeLocalStorage.setItem(JobsComponent.LOCAL_STORAGE_TYPE, this.jobType) |
90 | } | 91 | } |
91 | } | 92 | } |
diff --git a/client/src/app/core/server/server.service.ts b/client/src/app/core/server/server.service.ts index cdcbcb528..1f6cfb596 100644 --- a/client/src/app/core/server/server.service.ts +++ b/client/src/app/core/server/server.service.ts | |||
@@ -9,6 +9,7 @@ import { VideoConstant } from '../../../../../shared/models/videos' | |||
9 | import { isDefaultLocale, peertubeTranslate } from '../../../../../shared/models/i18n' | 9 | import { isDefaultLocale, peertubeTranslate } from '../../../../../shared/models/i18n' |
10 | import { getDevLocale, isOnDevLocale } from '@app/shared/i18n/i18n-utils' | 10 | import { getDevLocale, isOnDevLocale } from '@app/shared/i18n/i18n-utils' |
11 | import { sortBy } from '@app/shared/misc/utils' | 11 | import { sortBy } from '@app/shared/misc/utils' |
12 | import { ServerStats } from '@shared/models/server' | ||
12 | 13 | ||
13 | @Injectable() | 14 | @Injectable() |
14 | export class ServerService { | 15 | export class ServerService { |
@@ -16,6 +17,8 @@ export class ServerService { | |||
16 | private static BASE_VIDEO_URL = environment.apiUrl + '/api/v1/videos/' | 17 | private static BASE_VIDEO_URL = environment.apiUrl + '/api/v1/videos/' |
17 | private static BASE_VIDEO_PLAYLIST_URL = environment.apiUrl + '/api/v1/video-playlists/' | 18 | private static BASE_VIDEO_PLAYLIST_URL = environment.apiUrl + '/api/v1/video-playlists/' |
18 | private static BASE_LOCALE_URL = environment.apiUrl + '/client/locales/' | 19 | private static BASE_LOCALE_URL = environment.apiUrl + '/client/locales/' |
20 | private static BASE_STATS_URL = environment.apiUrl + '/api/v1/server/stats' | ||
21 | |||
19 | private static CONFIG_LOCAL_STORAGE_KEY = 'server-config' | 22 | private static CONFIG_LOCAL_STORAGE_KEY = 'server-config' |
20 | 23 | ||
21 | configReloaded = new Subject<void>() | 24 | configReloaded = new Subject<void>() |
@@ -235,6 +238,10 @@ export class ServerService { | |||
235 | return this.localeObservable.pipe(first()) | 238 | return this.localeObservable.pipe(first()) |
236 | } | 239 | } |
237 | 240 | ||
241 | getServerStats () { | ||
242 | return this.http.get<ServerStats>(ServerService.BASE_STATS_URL) | ||
243 | } | ||
244 | |||
238 | private loadAttributeEnum <T extends string | number> ( | 245 | private loadAttributeEnum <T extends string | number> ( |
239 | baseUrl: string, | 246 | baseUrl: string, |
240 | attributeName: 'categories' | 'licences' | 'languages' | 'privacies', | 247 | attributeName: 'categories' | 'licences' | 'languages' | 'privacies', |
diff --git a/client/src/app/shared/images/global-icon.component.ts b/client/src/app/shared/images/global-icon.component.ts index 806aca347..b6e641228 100644 --- a/client/src/app/shared/images/global-icon.component.ts +++ b/client/src/app/shared/images/global-icon.component.ts | |||
@@ -1,6 +1,5 @@ | |||
1 | import { ChangeDetectionStrategy, Component, ElementRef, Input, OnInit } from '@angular/core' | 1 | import { ChangeDetectionStrategy, Component, ElementRef, Input, OnInit } from '@angular/core' |
2 | import { HooksService } from '@app/core/plugins/hooks.service' | 2 | import { HooksService } from '@app/core/plugins/hooks.service' |
3 | import { I18n } from '@ngx-translate/i18n-polyfill' | ||
4 | 3 | ||
5 | const icons = { | 4 | const icons = { |
6 | 'add': require('!!raw-loader?!../../../assets/images/global/add.svg'), | 5 | 'add': require('!!raw-loader?!../../../assets/images/global/add.svg'), |
diff --git a/client/src/app/shared/instance/instance-statistics.component.ts b/client/src/app/shared/instance/instance-statistics.component.ts index 8ec728f05..40aa8a4c0 100644 --- a/client/src/app/shared/instance/instance-statistics.component.ts +++ b/client/src/app/shared/instance/instance-statistics.component.ts | |||
@@ -1,9 +1,6 @@ | |||
1 | import { map } from 'rxjs/operators' | ||
2 | import { HttpClient } from '@angular/common/http' | ||
3 | import { Component, OnInit } from '@angular/core' | 1 | import { Component, OnInit } from '@angular/core' |
4 | import { I18n } from '@ngx-translate/i18n-polyfill' | ||
5 | import { ServerStats } from '@shared/models/server' | 2 | import { ServerStats } from '@shared/models/server' |
6 | import { environment } from '../../../environments/environment' | 3 | import { ServerService } from '@app/core' |
7 | 4 | ||
8 | @Component({ | 5 | @Component({ |
9 | selector: 'my-instance-statistics', | 6 | selector: 'my-instance-statistics', |
@@ -11,27 +8,15 @@ import { environment } from '../../../environments/environment' | |||
11 | styleUrls: [ './instance-statistics.component.scss' ] | 8 | styleUrls: [ './instance-statistics.component.scss' ] |
12 | }) | 9 | }) |
13 | export class InstanceStatisticsComponent implements OnInit { | 10 | export class InstanceStatisticsComponent implements OnInit { |
14 | private static BASE_STATS_URL = environment.apiUrl + '/api/v1/server/stats' | ||
15 | |||
16 | serverStats: ServerStats = null | 11 | serverStats: ServerStats = null |
17 | 12 | ||
18 | constructor ( | 13 | constructor ( |
19 | private http: HttpClient, | 14 | private serverService: ServerService |
20 | private i18n: I18n | ||
21 | ) { | 15 | ) { |
22 | } | 16 | } |
23 | 17 | ||
24 | ngOnInit () { | 18 | ngOnInit () { |
25 | this.getStats() | 19 | this.serverService.getServerStats() |
26 | .subscribe( | 20 | .subscribe(res => this.serverStats = res) |
27 | res => { | ||
28 | this.serverStats = res | ||
29 | } | ||
30 | ) | ||
31 | } | ||
32 | |||
33 | getStats () { | ||
34 | return this.http | ||
35 | .get<ServerStats>(InstanceStatisticsComponent.BASE_STATS_URL) | ||
36 | } | 21 | } |
37 | } | 22 | } |
diff --git a/client/src/app/shared/shared.module.ts b/client/src/app/shared/shared.module.ts index b2eb13f73..d06d37d8c 100644 --- a/client/src/app/shared/shared.module.ts +++ b/client/src/app/shared/shared.module.ts | |||
@@ -98,6 +98,7 @@ import { FollowService } from '@app/shared/instance/follow.service' | |||
98 | import { MultiSelectModule } from 'primeng/multiselect' | 98 | import { MultiSelectModule } from 'primeng/multiselect' |
99 | import { FeatureBooleanComponent } from '@app/shared/instance/feature-boolean.component' | 99 | import { FeatureBooleanComponent } from '@app/shared/instance/feature-boolean.component' |
100 | import { InputReadonlyCopyComponent } from '@app/shared/forms/input-readonly-copy.component' | 100 | import { InputReadonlyCopyComponent } from '@app/shared/forms/input-readonly-copy.component' |
101 | import { RedundancyService } from '@app/shared/video/redundancy.service' | ||
101 | 102 | ||
102 | @NgModule({ | 103 | @NgModule({ |
103 | imports: [ | 104 | imports: [ |
@@ -300,6 +301,7 @@ import { InputReadonlyCopyComponent } from '@app/shared/forms/input-readonly-cop | |||
300 | UserNotificationService, | 301 | UserNotificationService, |
301 | 302 | ||
302 | FollowService, | 303 | FollowService, |
304 | RedundancyService, | ||
303 | 305 | ||
304 | I18n | 306 | I18n |
305 | ] | 307 | ] |
diff --git a/client/src/app/shared/video/redundancy.service.ts b/client/src/app/shared/video/redundancy.service.ts new file mode 100644 index 000000000..fb918d73b --- /dev/null +++ b/client/src/app/shared/video/redundancy.service.ts | |||
@@ -0,0 +1,73 @@ | |||
1 | import { catchError, map, toArray } from 'rxjs/operators' | ||
2 | import { HttpClient, HttpParams } from '@angular/common/http' | ||
3 | import { Injectable } from '@angular/core' | ||
4 | import { RestExtractor, RestPagination, RestService } from '@app/shared/rest' | ||
5 | import { SortMeta } from 'primeng/api' | ||
6 | import { ResultList, Video, VideoRedundanciesTarget, VideoRedundancy } from '@shared/models' | ||
7 | import { concat, Observable } from 'rxjs' | ||
8 | import { environment } from '../../../environments/environment' | ||
9 | |||
10 | @Injectable() | ||
11 | export class RedundancyService { | ||
12 | static BASE_REDUNDANCY_URL = environment.apiUrl + '/api/v1/server/redundancy' | ||
13 | |||
14 | constructor ( | ||
15 | private authHttp: HttpClient, | ||
16 | private restService: RestService, | ||
17 | private restExtractor: RestExtractor | ||
18 | ) { } | ||
19 | |||
20 | updateRedundancy (host: string, redundancyAllowed: boolean) { | ||
21 | const url = RedundancyService.BASE_REDUNDANCY_URL + '/' + host | ||
22 | |||
23 | const body = { redundancyAllowed } | ||
24 | |||
25 | return this.authHttp.put(url, body) | ||
26 | .pipe( | ||
27 | map(this.restExtractor.extractDataBool), | ||
28 | catchError(err => this.restExtractor.handleError(err)) | ||
29 | ) | ||
30 | } | ||
31 | |||
32 | listVideoRedundancies (options: { | ||
33 | pagination: RestPagination, | ||
34 | sort: SortMeta, | ||
35 | target?: VideoRedundanciesTarget | ||
36 | }): Observable<ResultList<VideoRedundancy>> { | ||
37 | const { pagination, sort, target } = options | ||
38 | |||
39 | let params = new HttpParams() | ||
40 | params = this.restService.addRestGetParams(params, pagination, sort) | ||
41 | |||
42 | if (target) params = params.append('target', target) | ||
43 | |||
44 | return this.authHttp.get<ResultList<VideoRedundancy>>(RedundancyService.BASE_REDUNDANCY_URL + '/videos', { params }) | ||
45 | .pipe( | ||
46 | catchError(res => this.restExtractor.handleError(res)) | ||
47 | ) | ||
48 | } | ||
49 | |||
50 | addVideoRedundancy (video: Video) { | ||
51 | return this.authHttp.post(RedundancyService.BASE_REDUNDANCY_URL + '/videos', { videoId: video.id }) | ||
52 | .pipe( | ||
53 | catchError(res => this.restExtractor.handleError(res)) | ||
54 | ) | ||
55 | } | ||
56 | |||
57 | removeVideoRedundancies (redundancy: VideoRedundancy) { | ||
58 | const observables = redundancy.redundancies.streamingPlaylists.map(r => r.id) | ||
59 | .concat(redundancy.redundancies.files.map(r => r.id)) | ||
60 | .map(id => this.removeRedundancy(id)) | ||
61 | |||
62 | return concat(...observables) | ||
63 | .pipe(toArray()) | ||
64 | } | ||
65 | |||
66 | private removeRedundancy (redundancyId: number) { | ||
67 | return this.authHttp.delete(RedundancyService.BASE_REDUNDANCY_URL + '/videos/' + redundancyId) | ||
68 | .pipe( | ||
69 | map(this.restExtractor.extractDataBool), | ||
70 | catchError(res => this.restExtractor.handleError(res)) | ||
71 | ) | ||
72 | } | ||
73 | } | ||
diff --git a/client/src/app/shared/video/video-actions-dropdown.component.ts b/client/src/app/shared/video/video-actions-dropdown.component.ts index afdeab18d..390d74c52 100644 --- a/client/src/app/shared/video/video-actions-dropdown.component.ts +++ b/client/src/app/shared/video/video-actions-dropdown.component.ts | |||
@@ -14,6 +14,7 @@ import { VideoBlacklistComponent } from '@app/shared/video/modals/video-blacklis | |||
14 | import { VideoBlacklistService } from '@app/shared/video-blacklist' | 14 | import { VideoBlacklistService } from '@app/shared/video-blacklist' |
15 | import { ScreenService } from '@app/shared/misc/screen.service' | 15 | import { ScreenService } from '@app/shared/misc/screen.service' |
16 | import { VideoCaption } from '@shared/models' | 16 | import { VideoCaption } from '@shared/models' |
17 | import { RedundancyService } from '@app/shared/video/redundancy.service' | ||
17 | 18 | ||
18 | export type VideoActionsDisplayType = { | 19 | export type VideoActionsDisplayType = { |
19 | playlist?: boolean | 20 | playlist?: boolean |
@@ -22,6 +23,7 @@ export type VideoActionsDisplayType = { | |||
22 | blacklist?: boolean | 23 | blacklist?: boolean |
23 | delete?: boolean | 24 | delete?: boolean |
24 | report?: boolean | 25 | report?: boolean |
26 | duplicate?: boolean | ||
25 | } | 27 | } |
26 | 28 | ||
27 | @Component({ | 29 | @Component({ |
@@ -46,7 +48,8 @@ export class VideoActionsDropdownComponent implements OnChanges { | |||
46 | update: true, | 48 | update: true, |
47 | blacklist: true, | 49 | blacklist: true, |
48 | delete: true, | 50 | delete: true, |
49 | report: true | 51 | report: true, |
52 | duplicate: true | ||
50 | } | 53 | } |
51 | @Input() placement = 'left' | 54 | @Input() placement = 'left' |
52 | 55 | ||
@@ -74,6 +77,7 @@ export class VideoActionsDropdownComponent implements OnChanges { | |||
74 | private screenService: ScreenService, | 77 | private screenService: ScreenService, |
75 | private videoService: VideoService, | 78 | private videoService: VideoService, |
76 | private blocklistService: BlocklistService, | 79 | private blocklistService: BlocklistService, |
80 | private redundancyService: RedundancyService, | ||
77 | private i18n: I18n | 81 | private i18n: I18n |
78 | ) { } | 82 | ) { } |
79 | 83 | ||
@@ -144,6 +148,10 @@ export class VideoActionsDropdownComponent implements OnChanges { | |||
144 | return this.video && this.video instanceof VideoDetails && this.video.downloadEnabled | 148 | return this.video && this.video instanceof VideoDetails && this.video.downloadEnabled |
145 | } | 149 | } |
146 | 150 | ||
151 | canVideoBeDuplicated () { | ||
152 | return this.video.canBeDuplicatedBy(this.user) | ||
153 | } | ||
154 | |||
147 | /* Action handlers */ | 155 | /* Action handlers */ |
148 | 156 | ||
149 | async unblacklistVideo () { | 157 | async unblacklistVideo () { |
@@ -186,6 +194,18 @@ export class VideoActionsDropdownComponent implements OnChanges { | |||
186 | ) | 194 | ) |
187 | } | 195 | } |
188 | 196 | ||
197 | duplicateVideo () { | ||
198 | this.redundancyService.addVideoRedundancy(this.video) | ||
199 | .subscribe( | ||
200 | () => { | ||
201 | const message = this.i18n('This video will be duplicated by your instance.') | ||
202 | this.notifier.success(message) | ||
203 | }, | ||
204 | |||
205 | err => this.notifier.error(err.message) | ||
206 | ) | ||
207 | } | ||
208 | |||
189 | onVideoBlacklisted () { | 209 | onVideoBlacklisted () { |
190 | this.videoBlacklisted.emit() | 210 | this.videoBlacklisted.emit() |
191 | } | 211 | } |
@@ -234,6 +254,12 @@ export class VideoActionsDropdownComponent implements OnChanges { | |||
234 | isDisplayed: () => this.authService.isLoggedIn() && this.displayOptions.blacklist && this.isVideoUnblacklistable() | 254 | isDisplayed: () => this.authService.isLoggedIn() && this.displayOptions.blacklist && this.isVideoUnblacklistable() |
235 | }, | 255 | }, |
236 | { | 256 | { |
257 | label: this.i18n('Duplicate (redundancy)'), | ||
258 | handler: () => this.duplicateVideo(), | ||
259 | isDisplayed: () => this.authService.isLoggedIn() && this.displayOptions.duplicate && this.canVideoBeDuplicated(), | ||
260 | iconName: 'cloud-download' | ||
261 | }, | ||
262 | { | ||
237 | label: this.i18n('Delete'), | 263 | label: this.i18n('Delete'), |
238 | handler: () => this.removeVideo(), | 264 | handler: () => this.removeVideo(), |
239 | isDisplayed: () => this.authService.isLoggedIn() && this.displayOptions.delete && this.isVideoRemovable(), | 265 | isDisplayed: () => this.authService.isLoggedIn() && this.displayOptions.delete && this.isVideoRemovable(), |
diff --git a/client/src/app/shared/video/video-miniature.component.ts b/client/src/app/shared/video/video-miniature.component.ts index 598a7a983..1dfb3eec7 100644 --- a/client/src/app/shared/video/video-miniature.component.ts +++ b/client/src/app/shared/video/video-miniature.component.ts | |||
@@ -64,7 +64,8 @@ export class VideoMiniatureComponent implements OnInit { | |||
64 | update: true, | 64 | update: true, |
65 | blacklist: true, | 65 | blacklist: true, |
66 | delete: true, | 66 | delete: true, |
67 | report: true | 67 | report: true, |
68 | duplicate: false | ||
68 | } | 69 | } |
69 | showActions = false | 70 | showActions = false |
70 | serverConfig: ServerConfig | 71 | serverConfig: ServerConfig |
diff --git a/client/src/app/shared/video/video.model.ts b/client/src/app/shared/video/video.model.ts index fb98d5382..9eeaf41b0 100644 --- a/client/src/app/shared/video/video.model.ts +++ b/client/src/app/shared/video/video.model.ts | |||
@@ -152,4 +152,8 @@ export class Video implements VideoServerModel { | |||
152 | isUpdatableBy (user: AuthUser) { | 152 | isUpdatableBy (user: AuthUser) { |
153 | return user && this.isLocal === true && (this.account.name === user.username || user.hasRight(UserRight.UPDATE_ANY_VIDEO)) | 153 | return user && this.isLocal === true && (this.account.name === user.username || user.hasRight(UserRight.UPDATE_ANY_VIDEO)) |
154 | } | 154 | } |
155 | |||
156 | canBeDuplicatedBy (user: AuthUser) { | ||
157 | return user && this.isLocal === false && user.hasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES) | ||
158 | } | ||
155 | } | 159 | } |
diff --git a/client/yarn.lock b/client/yarn.lock index 0855a2570..c900ae549 100644 --- a/client/yarn.lock +++ b/client/yarn.lock | |||
@@ -2586,6 +2586,29 @@ chardet@^0.7.0: | |||
2586 | resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e" | 2586 | resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e" |
2587 | integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA== | 2587 | integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA== |
2588 | 2588 | ||
2589 | chart.js@^2.9.3: | ||
2590 | version "2.9.3" | ||
2591 | resolved "https://registry.yarnpkg.com/chart.js/-/chart.js-2.9.3.tgz#ae3884114dafd381bc600f5b35a189138aac1ef7" | ||
2592 | integrity sha512-+2jlOobSk52c1VU6fzkh3UwqHMdSlgH1xFv9FKMqHiNCpXsGPQa/+81AFa+i3jZ253Mq9aAycPwDjnn1XbRNNw== | ||
2593 | dependencies: | ||
2594 | chartjs-color "^2.1.0" | ||
2595 | moment "^2.10.2" | ||
2596 | |||
2597 | chartjs-color-string@^0.6.0: | ||
2598 | version "0.6.0" | ||
2599 | resolved "https://registry.yarnpkg.com/chartjs-color-string/-/chartjs-color-string-0.6.0.tgz#1df096621c0e70720a64f4135ea171d051402f71" | ||
2600 | integrity sha512-TIB5OKn1hPJvO7JcteW4WY/63v6KwEdt6udfnDE9iCAZgy+V4SrbSxoIbTw/xkUIapjEI4ExGtD0+6D3KyFd7A== | ||
2601 | dependencies: | ||
2602 | color-name "^1.0.0" | ||
2603 | |||
2604 | chartjs-color@^2.1.0: | ||
2605 | version "2.4.1" | ||
2606 | resolved "https://registry.yarnpkg.com/chartjs-color/-/chartjs-color-2.4.1.tgz#6118bba202fe1ea79dd7f7c0f9da93467296c3b0" | ||
2607 | integrity sha512-haqOg1+Yebys/Ts/9bLo/BqUcONQOdr/hoEr2LLTRl6C5LXctUdHxsCYfvQVg5JIxITrfCNUDr4ntqmQk9+/0w== | ||
2608 | dependencies: | ||
2609 | chartjs-color-string "^0.6.0" | ||
2610 | color-convert "^1.9.3" | ||
2611 | |||
2589 | check-types@^8.0.3: | 2612 | check-types@^8.0.3: |
2590 | version "8.0.3" | 2613 | version "8.0.3" |
2591 | resolved "https://registry.yarnpkg.com/check-types/-/check-types-8.0.3.tgz#3356cca19c889544f2d7a95ed49ce508a0ecf552" | 2614 | resolved "https://registry.yarnpkg.com/check-types/-/check-types-8.0.3.tgz#3356cca19c889544f2d7a95ed49ce508a0ecf552" |
@@ -2800,7 +2823,7 @@ collection-visit@^1.0.0: | |||
2800 | map-visit "^1.0.0" | 2823 | map-visit "^1.0.0" |
2801 | object-visit "^1.0.0" | 2824 | object-visit "^1.0.0" |
2802 | 2825 | ||
2803 | color-convert@^1.9.0: | 2826 | color-convert@^1.9.0, color-convert@^1.9.3: |
2804 | version "1.9.3" | 2827 | version "1.9.3" |
2805 | resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" | 2828 | resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" |
2806 | integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== | 2829 | integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== |
@@ -2812,6 +2835,11 @@ color-name@1.1.3: | |||
2812 | resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" | 2835 | resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" |
2813 | integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= | 2836 | integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= |
2814 | 2837 | ||
2838 | color-name@^1.0.0: | ||
2839 | version "1.1.4" | ||
2840 | resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" | ||
2841 | integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== | ||
2842 | |||
2815 | colors@1.1.2: | 2843 | colors@1.1.2: |
2816 | version "1.1.2" | 2844 | version "1.1.2" |
2817 | resolved "https://registry.yarnpkg.com/colors/-/colors-1.1.2.tgz#168a4701756b6a7f51a12ce0c97bfa28c084ed63" | 2845 | resolved "https://registry.yarnpkg.com/colors/-/colors-1.1.2.tgz#168a4701756b6a7f51a12ce0c97bfa28c084ed63" |
@@ -6941,6 +6969,11 @@ mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1: | |||
6941 | dependencies: | 6969 | dependencies: |
6942 | minimist "0.0.8" | 6970 | minimist "0.0.8" |
6943 | 6971 | ||
6972 | moment@^2.10.2: | ||
6973 | version "2.24.0" | ||
6974 | resolved "https://registry.yarnpkg.com/moment/-/moment-2.24.0.tgz#0d055d53f5052aa653c9f6eb68bb5d12bf5c2b5b" | ||
6975 | integrity sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg== | ||
6976 | |||
6944 | mousetrap@^1.6.0: | 6977 | mousetrap@^1.6.0: |
6945 | version "1.6.3" | 6978 | version "1.6.3" |
6946 | resolved "https://registry.yarnpkg.com/mousetrap/-/mousetrap-1.6.3.tgz#80fee49665fd478bccf072c9d46bdf1bfed3558a" | 6979 | resolved "https://registry.yarnpkg.com/mousetrap/-/mousetrap-1.6.3.tgz#80fee49665fd478bccf072c9d46bdf1bfed3558a" |
diff --git a/config/test.yaml b/config/test.yaml index 5758c1887..74979f3a7 100644 --- a/config/test.yaml +++ b/config/test.yaml | |||
@@ -40,18 +40,18 @@ contact_form: | |||
40 | 40 | ||
41 | redundancy: | 41 | redundancy: |
42 | videos: | 42 | videos: |
43 | check_interval: '10 minutes' | 43 | check_interval: '1 minute' |
44 | strategies: | 44 | strategies: |
45 | - | 45 | - |
46 | size: '10MB' | 46 | size: '1000MB' |
47 | min_lifetime: '10 minutes' | 47 | min_lifetime: '10 minutes' |
48 | strategy: 'most-views' | 48 | strategy: 'most-views' |
49 | - | 49 | - |
50 | size: '10MB' | 50 | size: '1000MB' |
51 | min_lifetime: '10 minutes' | 51 | min_lifetime: '10 minutes' |
52 | strategy: 'trending' | 52 | strategy: 'trending' |
53 | - | 53 | - |
54 | size: '10MB' | 54 | size: '1000MB' |
55 | min_lifetime: '10 minutes' | 55 | min_lifetime: '10 minutes' |
56 | strategy: 'recently-added' | 56 | strategy: 'recently-added' |
57 | min_views: 1 | 57 | min_views: 1 |
diff --git a/package.json b/package.json index d4858725a..fed390c82 100644 --- a/package.json +++ b/package.json | |||
@@ -41,7 +41,7 @@ | |||
41 | "i18n:create-custom-files": "node ./dist/scripts/i18n/create-custom-files.js", | 41 | "i18n:create-custom-files": "node ./dist/scripts/i18n/create-custom-files.js", |
42 | "reset-password": "node ./dist/scripts/reset-password.js", | 42 | "reset-password": "node ./dist/scripts/reset-password.js", |
43 | "play": "scripty", | 43 | "play": "scripty", |
44 | "dev": "scripty", | 44 | "dev": "sh ./scripts/dev/index.sh", |
45 | "dev:server": "sh ./scripts/dev/server.sh", | 45 | "dev:server": "sh ./scripts/dev/server.sh", |
46 | "dev:embed": "scripty", | 46 | "dev:embed": "scripty", |
47 | "dev:client": "sh ./scripts/dev/client.sh", | 47 | "dev:client": "sh ./scripts/dev/client.sh", |
diff --git a/scripts/dev/index.sh b/scripts/dev/index.sh index d221d2fc8..9e89516b8 100755 --- a/scripts/dev/index.sh +++ b/scripts/dev/index.sh | |||
@@ -3,5 +3,5 @@ | |||
3 | set -eu | 3 | set -eu |
4 | 4 | ||
5 | NODE_ENV=test npm run concurrently -- -k \ | 5 | NODE_ENV=test npm run concurrently -- -k \ |
6 | "npm run dev:client -- --skip-server" \ | 6 | "sh scripts/dev/client.sh --skip-server" \ |
7 | "npm run dev:server" | 7 | "sh scripts/dev/server.sh" |
diff --git a/server/controllers/api/server/follows.ts b/server/controllers/api/server/follows.ts index 29a403a43..c69de21fb 100644 --- a/server/controllers/api/server/follows.ts +++ b/server/controllers/api/server/follows.ts | |||
@@ -24,7 +24,7 @@ import { | |||
24 | } from '../../../middlewares/validators' | 24 | } from '../../../middlewares/validators' |
25 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' | 25 | import { ActorFollowModel } from '../../../models/activitypub/actor-follow' |
26 | import { JobQueue } from '../../../lib/job-queue' | 26 | import { JobQueue } from '../../../lib/job-queue' |
27 | import { removeRedundancyOf } from '../../../lib/redundancy' | 27 | import { removeRedundanciesOfServer } from '../../../lib/redundancy' |
28 | import { sequelizeTypescript } from '../../../initializers/database' | 28 | import { sequelizeTypescript } from '../../../initializers/database' |
29 | import { autoFollowBackIfNeeded } from '../../../lib/activitypub/follow' | 29 | import { autoFollowBackIfNeeded } from '../../../lib/activitypub/follow' |
30 | 30 | ||
@@ -153,7 +153,7 @@ async function removeFollowing (req: express.Request, res: express.Response) { | |||
153 | await server.save({ transaction: t }) | 153 | await server.save({ transaction: t }) |
154 | 154 | ||
155 | // Async, could be long | 155 | // Async, could be long |
156 | removeRedundancyOf(server.id) | 156 | removeRedundanciesOfServer(server.id) |
157 | .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, err)) | 157 | .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, err)) |
158 | 158 | ||
159 | await follow.destroy({ transaction: t }) | 159 | await follow.destroy({ transaction: t }) |
diff --git a/server/controllers/api/server/redundancy.ts b/server/controllers/api/server/redundancy.ts index 4ea6164a3..a11c1a74f 100644 --- a/server/controllers/api/server/redundancy.ts +++ b/server/controllers/api/server/redundancy.ts | |||
@@ -1,9 +1,24 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import { UserRight } from '../../../../shared/models/users' | 2 | import { UserRight } from '../../../../shared/models/users' |
3 | import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares' | 3 | import { |
4 | import { updateServerRedundancyValidator } from '../../../middlewares/validators/redundancy' | 4 | asyncMiddleware, |
5 | import { removeRedundancyOf } from '../../../lib/redundancy' | 5 | authenticate, |
6 | ensureUserHasRight, | ||
7 | paginationValidator, | ||
8 | setDefaultPagination, | ||
9 | setDefaultVideoRedundanciesSort, | ||
10 | videoRedundanciesSortValidator | ||
11 | } from '../../../middlewares' | ||
12 | import { | ||
13 | listVideoRedundanciesValidator, | ||
14 | updateServerRedundancyValidator, | ||
15 | addVideoRedundancyValidator, | ||
16 | removeVideoRedundancyValidator | ||
17 | } from '../../../middlewares/validators/redundancy' | ||
18 | import { removeRedundanciesOfServer, removeVideoRedundancy } from '../../../lib/redundancy' | ||
6 | import { logger } from '../../../helpers/logger' | 19 | import { logger } from '../../../helpers/logger' |
20 | import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy' | ||
21 | import { JobQueue } from '@server/lib/job-queue' | ||
7 | 22 | ||
8 | const serverRedundancyRouter = express.Router() | 23 | const serverRedundancyRouter = express.Router() |
9 | 24 | ||
@@ -14,6 +29,31 @@ serverRedundancyRouter.put('/redundancy/:host', | |||
14 | asyncMiddleware(updateRedundancy) | 29 | asyncMiddleware(updateRedundancy) |
15 | ) | 30 | ) |
16 | 31 | ||
32 | serverRedundancyRouter.get('/redundancy/videos', | ||
33 | authenticate, | ||
34 | ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES), | ||
35 | listVideoRedundanciesValidator, | ||
36 | paginationValidator, | ||
37 | videoRedundanciesSortValidator, | ||
38 | setDefaultVideoRedundanciesSort, | ||
39 | setDefaultPagination, | ||
40 | asyncMiddleware(listVideoRedundancies) | ||
41 | ) | ||
42 | |||
43 | serverRedundancyRouter.post('/redundancy/videos', | ||
44 | authenticate, | ||
45 | ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES), | ||
46 | addVideoRedundancyValidator, | ||
47 | asyncMiddleware(addVideoRedundancy) | ||
48 | ) | ||
49 | |||
50 | serverRedundancyRouter.delete('/redundancy/videos/:redundancyId', | ||
51 | authenticate, | ||
52 | ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES), | ||
53 | removeVideoRedundancyValidator, | ||
54 | asyncMiddleware(removeVideoRedundancyController) | ||
55 | ) | ||
56 | |||
17 | // --------------------------------------------------------------------------- | 57 | // --------------------------------------------------------------------------- |
18 | 58 | ||
19 | export { | 59 | export { |
@@ -22,6 +62,42 @@ export { | |||
22 | 62 | ||
23 | // --------------------------------------------------------------------------- | 63 | // --------------------------------------------------------------------------- |
24 | 64 | ||
65 | async function listVideoRedundancies (req: express.Request, res: express.Response) { | ||
66 | const resultList = await VideoRedundancyModel.listForApi({ | ||
67 | start: req.query.start, | ||
68 | count: req.query.count, | ||
69 | sort: req.query.sort, | ||
70 | target: req.query.target, | ||
71 | strategy: req.query.strategy | ||
72 | }) | ||
73 | |||
74 | const result = { | ||
75 | total: resultList.total, | ||
76 | data: resultList.data.map(r => VideoRedundancyModel.toFormattedJSONStatic(r)) | ||
77 | } | ||
78 | |||
79 | return res.json(result) | ||
80 | } | ||
81 | |||
82 | async function addVideoRedundancy (req: express.Request, res: express.Response) { | ||
83 | const payload = { | ||
84 | videoId: res.locals.onlyVideo.id | ||
85 | } | ||
86 | |||
87 | await JobQueue.Instance.createJob({ | ||
88 | type: 'video-redundancy', | ||
89 | payload | ||
90 | }) | ||
91 | |||
92 | return res.sendStatus(204) | ||
93 | } | ||
94 | |||
95 | async function removeVideoRedundancyController (req: express.Request, res: express.Response) { | ||
96 | await removeVideoRedundancy(res.locals.videoRedundancy) | ||
97 | |||
98 | return res.sendStatus(204) | ||
99 | } | ||
100 | |||
25 | async function updateRedundancy (req: express.Request, res: express.Response) { | 101 | async function updateRedundancy (req: express.Request, res: express.Response) { |
26 | const server = res.locals.server | 102 | const server = res.locals.server |
27 | 103 | ||
@@ -30,7 +106,7 @@ async function updateRedundancy (req: express.Request, res: express.Response) { | |||
30 | await server.save() | 106 | await server.save() |
31 | 107 | ||
32 | // Async, could be long | 108 | // Async, could be long |
33 | removeRedundancyOf(server.id) | 109 | removeRedundanciesOfServer(server.id) |
34 | .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, { err })) | 110 | .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, { err })) |
35 | 111 | ||
36 | return res.sendStatus(204) | 112 | return res.sendStatus(204) |
diff --git a/server/controllers/api/server/stats.ts b/server/controllers/api/server/stats.ts index 3616c074d..6d508a481 100644 --- a/server/controllers/api/server/stats.ts +++ b/server/controllers/api/server/stats.ts | |||
@@ -10,6 +10,7 @@ import { ROUTE_CACHE_LIFETIME } from '../../../initializers/constants' | |||
10 | import { cacheRoute } from '../../../middlewares/cache' | 10 | import { cacheRoute } from '../../../middlewares/cache' |
11 | import { VideoFileModel } from '../../../models/video/video-file' | 11 | import { VideoFileModel } from '../../../models/video/video-file' |
12 | import { CONFIG } from '../../../initializers/config' | 12 | import { CONFIG } from '../../../initializers/config' |
13 | import { VideoRedundancyStrategyWithManual } from '@shared/models' | ||
13 | 14 | ||
14 | const statsRouter = express.Router() | 15 | const statsRouter = express.Router() |
15 | 16 | ||
@@ -25,8 +26,15 @@ async function getStats (req: express.Request, res: express.Response) { | |||
25 | const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() | 26 | const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() |
26 | const { totalLocalVideoFilesSize } = await VideoFileModel.getStats() | 27 | const { totalLocalVideoFilesSize } = await VideoFileModel.getStats() |
27 | 28 | ||
29 | const strategies: { strategy: VideoRedundancyStrategyWithManual, size: number }[] = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES | ||
30 | .map(r => ({ | ||
31 | strategy: r.strategy, | ||
32 | size: r.size | ||
33 | })) | ||
34 | strategies.push({ strategy: 'manual', size: null }) | ||
35 | |||
28 | const videosRedundancyStats = await Promise.all( | 36 | const videosRedundancyStats = await Promise.all( |
29 | CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.map(r => { | 37 | strategies.map(r => { |
30 | return VideoRedundancyModel.getStats(r.strategy) | 38 | return VideoRedundancyModel.getStats(r.strategy) |
31 | .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size })) | 39 | .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size })) |
32 | }) | 40 | }) |
diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts index 21d5c53ca..c5b3b4d9f 100644 --- a/server/helpers/custom-validators/activitypub/cache-file.ts +++ b/server/helpers/custom-validators/activitypub/cache-file.ts | |||
@@ -6,7 +6,7 @@ import { CacheFileObject } from '../../../../shared/models/activitypub/objects' | |||
6 | function isCacheFileObjectValid (object: CacheFileObject) { | 6 | function isCacheFileObjectValid (object: CacheFileObject) { |
7 | return exists(object) && | 7 | return exists(object) && |
8 | object.type === 'CacheFile' && | 8 | object.type === 'CacheFile' && |
9 | isDateValid(object.expires) && | 9 | (object.expires === null || isDateValid(object.expires)) && |
10 | isActivityPubUrlValid(object.object) && | 10 | isActivityPubUrlValid(object.object) && |
11 | (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url)) | 11 | (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url)) |
12 | } | 12 | } |
diff --git a/server/helpers/custom-validators/video-redundancies.ts b/server/helpers/custom-validators/video-redundancies.ts new file mode 100644 index 000000000..50a559c4f --- /dev/null +++ b/server/helpers/custom-validators/video-redundancies.ts | |||
@@ -0,0 +1,12 @@ | |||
1 | import { exists } from './misc' | ||
2 | |||
3 | function isVideoRedundancyTarget (value: any) { | ||
4 | return exists(value) && | ||
5 | (value === 'my-videos' || value === 'remote-videos') | ||
6 | } | ||
7 | |||
8 | // --------------------------------------------------------------------------- | ||
9 | |||
10 | export { | ||
11 | isVideoRedundancyTarget | ||
12 | } | ||
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index 3a99518c6..8a5d030df 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts | |||
@@ -9,12 +9,12 @@ import { promisify2 } from './core-utils' | |||
9 | import { MVideo } from '@server/typings/models/video/video' | 9 | import { MVideo } from '@server/typings/models/video/video' |
10 | import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/typings/models/video/video-file' | 10 | import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/typings/models/video/video-file' |
11 | import { isStreamingPlaylist, MStreamingPlaylistVideo } from '@server/typings/models/video/video-streaming-playlist' | 11 | import { isStreamingPlaylist, MStreamingPlaylistVideo } from '@server/typings/models/video/video-streaming-playlist' |
12 | import { STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' | 12 | import { WEBSERVER } from '@server/initializers/constants' |
13 | import * as parseTorrent from 'parse-torrent' | 13 | import * as parseTorrent from 'parse-torrent' |
14 | import * as magnetUtil from 'magnet-uri' | 14 | import * as magnetUtil from 'magnet-uri' |
15 | import { isArray } from '@server/helpers/custom-validators/misc' | 15 | import { isArray } from '@server/helpers/custom-validators/misc' |
16 | import { extractVideo } from '@server/lib/videos' | 16 | import { extractVideo } from '@server/lib/videos' |
17 | import { getTorrentFileName, getVideoFilename, getVideoFilePath } from '@server/lib/video-paths' | 17 | import { getTorrentFileName, getVideoFilePath } from '@server/lib/video-paths' |
18 | 18 | ||
19 | const createTorrentPromise = promisify2<string, any, any>(createTorrent) | 19 | const createTorrentPromise = promisify2<string, any, any>(createTorrent) |
20 | 20 | ||
diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 7fd77f3e8..fd8bf09fc 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { IConfig } from 'config' | 1 | import { IConfig } from 'config' |
2 | import { dirname, join } from 'path' | 2 | import { dirname, join } from 'path' |
3 | import { VideosRedundancy } from '../../shared/models' | 3 | import { VideosRedundancyStrategy } from '../../shared/models' |
4 | // Do not use barrels, remain constants as independent as possible | 4 | // Do not use barrels, remain constants as independent as possible |
5 | import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils' | 5 | import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils' |
6 | import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' | 6 | import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' |
@@ -304,7 +304,7 @@ function getLocalConfigFilePath () { | |||
304 | return join(dirname(configSources[ 0 ].name), filename + '.json') | 304 | return join(dirname(configSources[ 0 ].name), filename + '.json') |
305 | } | 305 | } |
306 | 306 | ||
307 | function buildVideosRedundancy (objs: any[]): VideosRedundancy[] { | 307 | function buildVideosRedundancy (objs: any[]): VideosRedundancyStrategy[] { |
308 | if (!objs) return [] | 308 | if (!objs) return [] |
309 | 309 | ||
310 | if (!Array.isArray(objs)) return objs | 310 | if (!Array.isArray(objs)) return objs |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 032f63c8f..e01ab8943 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -14,7 +14,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' | |||
14 | 14 | ||
15 | // --------------------------------------------------------------------------- | 15 | // --------------------------------------------------------------------------- |
16 | 16 | ||
17 | const LAST_MIGRATION_VERSION = 470 | 17 | const LAST_MIGRATION_VERSION = 475 |
18 | 18 | ||
19 | // --------------------------------------------------------------------------- | 19 | // --------------------------------------------------------------------------- |
20 | 20 | ||
@@ -73,7 +73,9 @@ const SORTABLE_COLUMNS = { | |||
73 | 73 | ||
74 | PLUGINS: [ 'name', 'createdAt', 'updatedAt' ], | 74 | PLUGINS: [ 'name', 'createdAt', 'updatedAt' ], |
75 | 75 | ||
76 | AVAILABLE_PLUGINS: [ 'npmName', 'popularity' ] | 76 | AVAILABLE_PLUGINS: [ 'npmName', 'popularity' ], |
77 | |||
78 | VIDEO_REDUNDANCIES: [ 'name' ] | ||
77 | } | 79 | } |
78 | 80 | ||
79 | const OAUTH_LIFETIME = { | 81 | const OAUTH_LIFETIME = { |
@@ -117,45 +119,44 @@ const REMOTE_SCHEME = { | |||
117 | WS: 'wss' | 119 | WS: 'wss' |
118 | } | 120 | } |
119 | 121 | ||
120 | // TODO: remove 'video-file' | 122 | const JOB_ATTEMPTS: { [id in JobType]: number } = { |
121 | const JOB_ATTEMPTS: { [id in (JobType | 'video-file')]: number } = { | ||
122 | 'activitypub-http-broadcast': 5, | 123 | 'activitypub-http-broadcast': 5, |
123 | 'activitypub-http-unicast': 5, | 124 | 'activitypub-http-unicast': 5, |
124 | 'activitypub-http-fetcher': 5, | 125 | 'activitypub-http-fetcher': 5, |
125 | 'activitypub-follow': 5, | 126 | 'activitypub-follow': 5, |
126 | 'video-file-import': 1, | 127 | 'video-file-import': 1, |
127 | 'video-transcoding': 1, | 128 | 'video-transcoding': 1, |
128 | 'video-file': 1, | ||
129 | 'video-import': 1, | 129 | 'video-import': 1, |
130 | 'email': 5, | 130 | 'email': 5, |
131 | 'videos-views': 1, | 131 | 'videos-views': 1, |
132 | 'activitypub-refresher': 1 | 132 | 'activitypub-refresher': 1, |
133 | 'video-redundancy': 1 | ||
133 | } | 134 | } |
134 | const JOB_CONCURRENCY: { [id in (JobType | 'video-file')]: number } = { | 135 | const JOB_CONCURRENCY: { [id in JobType]: number } = { |
135 | 'activitypub-http-broadcast': 1, | 136 | 'activitypub-http-broadcast': 1, |
136 | 'activitypub-http-unicast': 5, | 137 | 'activitypub-http-unicast': 5, |
137 | 'activitypub-http-fetcher': 1, | 138 | 'activitypub-http-fetcher': 1, |
138 | 'activitypub-follow': 1, | 139 | 'activitypub-follow': 1, |
139 | 'video-file-import': 1, | 140 | 'video-file-import': 1, |
140 | 'video-transcoding': 1, | 141 | 'video-transcoding': 1, |
141 | 'video-file': 1, | ||
142 | 'video-import': 1, | 142 | 'video-import': 1, |
143 | 'email': 5, | 143 | 'email': 5, |
144 | 'videos-views': 1, | 144 | 'videos-views': 1, |
145 | 'activitypub-refresher': 1 | 145 | 'activitypub-refresher': 1, |
146 | 'video-redundancy': 1 | ||
146 | } | 147 | } |
147 | const JOB_TTL: { [id in (JobType | 'video-file')]: number } = { | 148 | const JOB_TTL: { [id in JobType]: number } = { |
148 | 'activitypub-http-broadcast': 60000 * 10, // 10 minutes | 149 | 'activitypub-http-broadcast': 60000 * 10, // 10 minutes |
149 | 'activitypub-http-unicast': 60000 * 10, // 10 minutes | 150 | 'activitypub-http-unicast': 60000 * 10, // 10 minutes |
150 | 'activitypub-http-fetcher': 60000 * 10, // 10 minutes | 151 | 'activitypub-http-fetcher': 60000 * 10, // 10 minutes |
151 | 'activitypub-follow': 60000 * 10, // 10 minutes | 152 | 'activitypub-follow': 60000 * 10, // 10 minutes |
152 | 'video-file-import': 1000 * 3600, // 1 hour | 153 | 'video-file-import': 1000 * 3600, // 1 hour |
153 | 'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long | 154 | 'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long |
154 | 'video-file': 1000 * 3600 * 48, // 2 days, transcoding could be long | ||
155 | 'video-import': 1000 * 3600 * 2, // hours | 155 | 'video-import': 1000 * 3600 * 2, // hours |
156 | 'email': 60000 * 10, // 10 minutes | 156 | 'email': 60000 * 10, // 10 minutes |
157 | 'videos-views': undefined, // Unlimited | 157 | 'videos-views': undefined, // Unlimited |
158 | 'activitypub-refresher': 60000 * 10 // 10 minutes | 158 | 'activitypub-refresher': 60000 * 10, // 10 minutes |
159 | 'video-redundancy': 1000 * 3600 * 3 // 3 hours | ||
159 | } | 160 | } |
160 | const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = { | 161 | const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = { |
161 | 'videos-views': { | 162 | 'videos-views': { |
diff --git a/server/initializers/migrations/0475-redundancy-expires-on.ts b/server/initializers/migrations/0475-redundancy-expires-on.ts new file mode 100644 index 000000000..7e392c8c0 --- /dev/null +++ b/server/initializers/migrations/0475-redundancy-expires-on.ts | |||
@@ -0,0 +1,27 @@ | |||
1 | import * as Sequelize from 'sequelize' | ||
2 | |||
3 | async function up (utils: { | ||
4 | transaction: Sequelize.Transaction, | ||
5 | queryInterface: Sequelize.QueryInterface, | ||
6 | sequelize: Sequelize.Sequelize, | ||
7 | db: any | ||
8 | }): Promise<void> { | ||
9 | { | ||
10 | const data = { | ||
11 | type: Sequelize.DATE, | ||
12 | allowNull: true, | ||
13 | defaultValue: null | ||
14 | } | ||
15 | |||
16 | await utils.queryInterface.changeColumn('videoRedundancy', 'expiresOn', data) | ||
17 | } | ||
18 | } | ||
19 | |||
20 | function down (options) { | ||
21 | throw new Error('Not implemented.') | ||
22 | } | ||
23 | |||
24 | export { | ||
25 | up, | ||
26 | down | ||
27 | } | ||
diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts index 65b2dcb49..8252e95e9 100644 --- a/server/lib/activitypub/cache-file.ts +++ b/server/lib/activitypub/cache-file.ts | |||
@@ -13,7 +13,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject | |||
13 | if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) | 13 | if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) |
14 | 14 | ||
15 | return { | 15 | return { |
16 | expiresOn: new Date(cacheFileObject.expires), | 16 | expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, |
17 | url: cacheFileObject.id, | 17 | url: cacheFileObject.id, |
18 | fileUrl: url.href, | 18 | fileUrl: url.href, |
19 | strategy: null, | 19 | strategy: null, |
@@ -30,7 +30,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject | |||
30 | if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`) | 30 | if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`) |
31 | 31 | ||
32 | return { | 32 | return { |
33 | expiresOn: new Date(cacheFileObject.expires), | 33 | expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, |
34 | url: cacheFileObject.id, | 34 | url: cacheFileObject.id, |
35 | fileUrl: url.href, | 35 | fileUrl: url.href, |
36 | strategy: null, | 36 | strategy: null, |
diff --git a/server/lib/job-queue/handlers/video-redundancy.ts b/server/lib/job-queue/handlers/video-redundancy.ts new file mode 100644 index 000000000..319d7090e --- /dev/null +++ b/server/lib/job-queue/handlers/video-redundancy.ts | |||
@@ -0,0 +1,20 @@ | |||
1 | import * as Bull from 'bull' | ||
2 | import { logger } from '../../../helpers/logger' | ||
3 | import { VideosRedundancyScheduler } from '@server/lib/schedulers/videos-redundancy-scheduler' | ||
4 | |||
5 | export type VideoRedundancyPayload = { | ||
6 | videoId: number | ||
7 | } | ||
8 | |||
9 | async function processVideoRedundancy (job: Bull.Job) { | ||
10 | const payload = job.data as VideoRedundancyPayload | ||
11 | logger.info('Processing video redundancy in job %d.', job.id) | ||
12 | |||
13 | return VideosRedundancyScheduler.Instance.createManualRedundancy(payload.videoId) | ||
14 | } | ||
15 | |||
16 | // --------------------------------------------------------------------------- | ||
17 | |||
18 | export { | ||
19 | processVideoRedundancy | ||
20 | } | ||
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index ec601e9ea..8bbf58f2b 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts | |||
@@ -13,6 +13,7 @@ import { processVideoImport, VideoImportPayload } from './handlers/video-import' | |||
13 | import { processVideosViews } from './handlers/video-views' | 13 | import { processVideosViews } from './handlers/video-views' |
14 | import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher' | 14 | import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher' |
15 | import { processVideoFileImport, VideoFileImportPayload } from './handlers/video-file-import' | 15 | import { processVideoFileImport, VideoFileImportPayload } from './handlers/video-file-import' |
16 | import { processVideoRedundancy, VideoRedundancyPayload } from '@server/lib/job-queue/handlers/video-redundancy' | ||
16 | 17 | ||
17 | type CreateJobArgument = | 18 | type CreateJobArgument = |
18 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | | 19 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | |
@@ -24,20 +25,21 @@ type CreateJobArgument = | |||
24 | { type: 'email', payload: EmailPayload } | | 25 | { type: 'email', payload: EmailPayload } | |
25 | { type: 'video-import', payload: VideoImportPayload } | | 26 | { type: 'video-import', payload: VideoImportPayload } | |
26 | { type: 'activitypub-refresher', payload: RefreshPayload } | | 27 | { type: 'activitypub-refresher', payload: RefreshPayload } | |
27 | { type: 'videos-views', payload: {} } | 28 | { type: 'videos-views', payload: {} } | |
29 | { type: 'video-redundancy', payload: VideoRedundancyPayload } | ||
28 | 30 | ||
29 | const handlers: { [ id in (JobType | 'video-file') ]: (job: Bull.Job) => Promise<any>} = { | 31 | const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = { |
30 | 'activitypub-http-broadcast': processActivityPubHttpBroadcast, | 32 | 'activitypub-http-broadcast': processActivityPubHttpBroadcast, |
31 | 'activitypub-http-unicast': processActivityPubHttpUnicast, | 33 | 'activitypub-http-unicast': processActivityPubHttpUnicast, |
32 | 'activitypub-http-fetcher': processActivityPubHttpFetcher, | 34 | 'activitypub-http-fetcher': processActivityPubHttpFetcher, |
33 | 'activitypub-follow': processActivityPubFollow, | 35 | 'activitypub-follow': processActivityPubFollow, |
34 | 'video-file-import': processVideoFileImport, | 36 | 'video-file-import': processVideoFileImport, |
35 | 'video-transcoding': processVideoTranscoding, | 37 | 'video-transcoding': processVideoTranscoding, |
36 | 'video-file': processVideoTranscoding, // TODO: remove it (changed in 1.3) | ||
37 | 'email': processEmail, | 38 | 'email': processEmail, |
38 | 'video-import': processVideoImport, | 39 | 'video-import': processVideoImport, |
39 | 'videos-views': processVideosViews, | 40 | 'videos-views': processVideosViews, |
40 | 'activitypub-refresher': refreshAPObject | 41 | 'activitypub-refresher': refreshAPObject, |
42 | 'video-redundancy': processVideoRedundancy | ||
41 | } | 43 | } |
42 | 44 | ||
43 | const jobTypes: JobType[] = [ | 45 | const jobTypes: JobType[] = [ |
@@ -50,7 +52,8 @@ const jobTypes: JobType[] = [ | |||
50 | 'video-file-import', | 52 | 'video-file-import', |
51 | 'video-import', | 53 | 'video-import', |
52 | 'videos-views', | 54 | 'videos-views', |
53 | 'activitypub-refresher' | 55 | 'activitypub-refresher', |
56 | 'video-redundancy' | ||
54 | ] | 57 | ] |
55 | 58 | ||
56 | class JobQueue { | 59 | class JobQueue { |
diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts index 1b4ecd7c0..78d84e02e 100644 --- a/server/lib/redundancy.ts +++ b/server/lib/redundancy.ts | |||
@@ -13,10 +13,10 @@ async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t? | |||
13 | await videoRedundancy.destroy({ transaction: t }) | 13 | await videoRedundancy.destroy({ transaction: t }) |
14 | } | 14 | } |
15 | 15 | ||
16 | async function removeRedundancyOf (serverId: number) { | 16 | async function removeRedundanciesOfServer (serverId: number) { |
17 | const videosRedundancy = await VideoRedundancyModel.listLocalOfServer(serverId) | 17 | const redundancies = await VideoRedundancyModel.listLocalOfServer(serverId) |
18 | 18 | ||
19 | for (const redundancy of videosRedundancy) { | 19 | for (const redundancy of redundancies) { |
20 | await removeVideoRedundancy(redundancy) | 20 | await removeVideoRedundancy(redundancy) |
21 | } | 21 | } |
22 | } | 22 | } |
@@ -24,6 +24,6 @@ async function removeRedundancyOf (serverId: number) { | |||
24 | // --------------------------------------------------------------------------- | 24 | // --------------------------------------------------------------------------- |
25 | 25 | ||
26 | export { | 26 | export { |
27 | removeRedundancyOf, | 27 | removeRedundanciesOfServer, |
28 | removeVideoRedundancy | 28 | removeVideoRedundancy |
29 | } | 29 | } |
diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts index 350a335d3..956780a77 100644 --- a/server/lib/schedulers/update-videos-scheduler.ts +++ b/server/lib/schedulers/update-videos-scheduler.ts | |||
@@ -4,7 +4,6 @@ import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-upda | |||
4 | import { retryTransactionWrapper } from '../../helpers/database-utils' | 4 | import { retryTransactionWrapper } from '../../helpers/database-utils' |
5 | import { federateVideoIfNeeded } from '../activitypub' | 5 | import { federateVideoIfNeeded } from '../activitypub' |
6 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' | 6 | import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' |
7 | import { VideoPrivacy } from '../../../shared/models/videos' | ||
8 | import { Notifier } from '../notifier' | 7 | import { Notifier } from '../notifier' |
9 | import { sequelizeTypescript } from '../../initializers/database' | 8 | import { sequelizeTypescript } from '../../initializers/database' |
10 | import { MVideoFullLight } from '@server/typings/models' | 9 | import { MVideoFullLight } from '@server/typings/models' |
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index c1c91b656..6e61cbe7d 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { AbstractScheduler } from './abstract-scheduler' | 1 | import { AbstractScheduler } from './abstract-scheduler' |
2 | import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER } from '../../initializers/constants' | 2 | import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER } from '../../initializers/constants' |
3 | import { logger } from '../../helpers/logger' | 3 | import { logger } from '../../helpers/logger' |
4 | import { VideosRedundancy } from '../../../shared/models/redundancy' | 4 | import { VideosRedundancyStrategy } from '../../../shared/models/redundancy' |
5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 5 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
6 | import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent' | 6 | import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent' |
7 | import { join } from 'path' | 7 | import { join } from 'path' |
@@ -25,9 +25,10 @@ import { | |||
25 | MVideoWithAllFiles | 25 | MVideoWithAllFiles |
26 | } from '@server/typings/models' | 26 | } from '@server/typings/models' |
27 | import { getVideoFilename } from '../video-paths' | 27 | import { getVideoFilename } from '../video-paths' |
28 | import { VideoModel } from '@server/models/video/video' | ||
28 | 29 | ||
29 | type CandidateToDuplicate = { | 30 | type CandidateToDuplicate = { |
30 | redundancy: VideosRedundancy, | 31 | redundancy: VideosRedundancyStrategy, |
31 | video: MVideoWithAllFiles, | 32 | video: MVideoWithAllFiles, |
32 | files: MVideoFile[], | 33 | files: MVideoFile[], |
33 | streamingPlaylists: MStreamingPlaylistFiles[] | 34 | streamingPlaylists: MStreamingPlaylistFiles[] |
@@ -41,7 +42,7 @@ function isMVideoRedundancyFileVideo ( | |||
41 | 42 | ||
42 | export class VideosRedundancyScheduler extends AbstractScheduler { | 43 | export class VideosRedundancyScheduler extends AbstractScheduler { |
43 | 44 | ||
44 | private static instance: AbstractScheduler | 45 | private static instance: VideosRedundancyScheduler |
45 | 46 | ||
46 | protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL | 47 | protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL |
47 | 48 | ||
@@ -49,6 +50,22 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
49 | super() | 50 | super() |
50 | } | 51 | } |
51 | 52 | ||
53 | async createManualRedundancy (videoId: number) { | ||
54 | const videoToDuplicate = await VideoModel.loadWithFiles(videoId) | ||
55 | |||
56 | if (!videoToDuplicate) { | ||
57 | logger.warn('Video to manually duplicate %d does not exist anymore.', videoId) | ||
58 | return | ||
59 | } | ||
60 | |||
61 | return this.createVideoRedundancies({ | ||
62 | video: videoToDuplicate, | ||
63 | redundancy: null, | ||
64 | files: videoToDuplicate.VideoFiles, | ||
65 | streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists | ||
66 | }) | ||
67 | } | ||
68 | |||
52 | protected async internalExecute () { | 69 | protected async internalExecute () { |
53 | for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { | 70 | for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { |
54 | logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy) | 71 | logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy) |
@@ -94,7 +111,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
94 | for (const redundancyModel of expired) { | 111 | for (const redundancyModel of expired) { |
95 | try { | 112 | try { |
96 | const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) | 113 | const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) |
97 | const candidate = { | 114 | const candidate: CandidateToDuplicate = { |
98 | redundancy: redundancyConfig, | 115 | redundancy: redundancyConfig, |
99 | video: null, | 116 | video: null, |
100 | files: [], | 117 | files: [], |
@@ -140,7 +157,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
140 | } | 157 | } |
141 | } | 158 | } |
142 | 159 | ||
143 | private findVideoToDuplicate (cache: VideosRedundancy) { | 160 | private findVideoToDuplicate (cache: VideosRedundancyStrategy) { |
144 | if (cache.strategy === 'most-views') { | 161 | if (cache.strategy === 'most-views') { |
145 | return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) | 162 | return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) |
146 | } | 163 | } |
@@ -187,13 +204,21 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
187 | } | 204 | } |
188 | } | 205 | } |
189 | 206 | ||
190 | private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: MVideoAccountLight, fileArg: MVideoFile) { | 207 | private async createVideoFileRedundancy (redundancy: VideosRedundancyStrategy | null, video: MVideoAccountLight, fileArg: MVideoFile) { |
208 | let strategy = 'manual' | ||
209 | let expiresOn: Date = null | ||
210 | |||
211 | if (redundancy) { | ||
212 | strategy = redundancy.strategy | ||
213 | expiresOn = this.buildNewExpiration(redundancy.minLifetime) | ||
214 | } | ||
215 | |||
191 | const file = fileArg as MVideoFileVideo | 216 | const file = fileArg as MVideoFileVideo |
192 | file.Video = video | 217 | file.Video = video |
193 | 218 | ||
194 | const serverActor = await getServerActor() | 219 | const serverActor = await getServerActor() |
195 | 220 | ||
196 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) | 221 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy) |
197 | 222 | ||
198 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() | 223 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() |
199 | const magnetUri = generateMagnetUri(video, file, baseUrlHttp, baseUrlWs) | 224 | const magnetUri = generateMagnetUri(video, file, baseUrlHttp, baseUrlWs) |
@@ -204,10 +229,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
204 | await move(tmpPath, destPath, { overwrite: true }) | 229 | await move(tmpPath, destPath, { overwrite: true }) |
205 | 230 | ||
206 | const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ | 231 | const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ |
207 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), | 232 | expiresOn, |
208 | url: getVideoCacheFileActivityPubUrl(file), | 233 | url: getVideoCacheFileActivityPubUrl(file), |
209 | fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL), | 234 | fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL), |
210 | strategy: redundancy.strategy, | 235 | strategy, |
211 | videoFileId: file.id, | 236 | videoFileId: file.id, |
212 | actorId: serverActor.id | 237 | actorId: serverActor.id |
213 | }) | 238 | }) |
@@ -220,25 +245,33 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
220 | } | 245 | } |
221 | 246 | ||
222 | private async createStreamingPlaylistRedundancy ( | 247 | private async createStreamingPlaylistRedundancy ( |
223 | redundancy: VideosRedundancy, | 248 | redundancy: VideosRedundancyStrategy, |
224 | video: MVideoAccountLight, | 249 | video: MVideoAccountLight, |
225 | playlistArg: MStreamingPlaylist | 250 | playlistArg: MStreamingPlaylist |
226 | ) { | 251 | ) { |
252 | let strategy = 'manual' | ||
253 | let expiresOn: Date = null | ||
254 | |||
255 | if (redundancy) { | ||
256 | strategy = redundancy.strategy | ||
257 | expiresOn = this.buildNewExpiration(redundancy.minLifetime) | ||
258 | } | ||
259 | |||
227 | const playlist = playlistArg as MStreamingPlaylistVideo | 260 | const playlist = playlistArg as MStreamingPlaylistVideo |
228 | playlist.Video = video | 261 | playlist.Video = video |
229 | 262 | ||
230 | const serverActor = await getServerActor() | 263 | const serverActor = await getServerActor() |
231 | 264 | ||
232 | logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy) | 265 | logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy) |
233 | 266 | ||
234 | const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) | 267 | const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid) |
235 | await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) | 268 | await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT) |
236 | 269 | ||
237 | const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ | 270 | const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ |
238 | expiresOn: this.buildNewExpiration(redundancy.minLifetime), | 271 | expiresOn, |
239 | url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), | 272 | url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), |
240 | fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL), | 273 | fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL), |
241 | strategy: redundancy.strategy, | 274 | strategy, |
242 | videoStreamingPlaylistId: playlist.id, | 275 | videoStreamingPlaylistId: playlist.id, |
243 | actorId: serverActor.id | 276 | actorId: serverActor.id |
244 | }) | 277 | }) |
diff --git a/server/middlewares/sort.ts b/server/middlewares/sort.ts index 8c27e8237..75238228f 100644 --- a/server/middlewares/sort.ts +++ b/server/middlewares/sort.ts | |||
@@ -1,17 +1,11 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import { SortType } from '../models/utils' | 2 | import { SortType } from '../models/utils' |
3 | 3 | ||
4 | function setDefaultSort (req: express.Request, res: express.Response, next: express.NextFunction) { | 4 | const setDefaultSort = setDefaultSortFactory('-createdAt') |
5 | if (!req.query.sort) req.query.sort = '-createdAt' | ||
6 | |||
7 | return next() | ||
8 | } | ||
9 | 5 | ||
10 | function setDefaultSearchSort (req: express.Request, res: express.Response, next: express.NextFunction) { | 6 | const setDefaultVideoRedundanciesSort = setDefaultSortFactory('name') |
11 | if (!req.query.sort) req.query.sort = '-match' | ||
12 | 7 | ||
13 | return next() | 8 | const setDefaultSearchSort = setDefaultSortFactory('-match') |
14 | } | ||
15 | 9 | ||
16 | function setBlacklistSort (req: express.Request, res: express.Response, next: express.NextFunction) { | 10 | function setBlacklistSort (req: express.Request, res: express.Response, next: express.NextFunction) { |
17 | let newSort: SortType = { sortModel: undefined, sortValue: '' } | 11 | let newSort: SortType = { sortModel: undefined, sortValue: '' } |
@@ -39,5 +33,16 @@ function setBlacklistSort (req: express.Request, res: express.Response, next: ex | |||
39 | export { | 33 | export { |
40 | setDefaultSort, | 34 | setDefaultSort, |
41 | setDefaultSearchSort, | 35 | setDefaultSearchSort, |
36 | setDefaultVideoRedundanciesSort, | ||
42 | setBlacklistSort | 37 | setBlacklistSort |
43 | } | 38 | } |
39 | |||
40 | // --------------------------------------------------------------------------- | ||
41 | |||
42 | function setDefaultSortFactory (sort: string) { | ||
43 | return (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
44 | if (!req.query.sort) req.query.sort = sort | ||
45 | |||
46 | return next() | ||
47 | } | ||
48 | } | ||
diff --git a/server/middlewares/validators/redundancy.ts b/server/middlewares/validators/redundancy.ts index 8098e3a44..16b42fc0d 100644 --- a/server/middlewares/validators/redundancy.ts +++ b/server/middlewares/validators/redundancy.ts | |||
@@ -1,12 +1,13 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import { body, param } from 'express-validator' | 2 | import { body, param, query } from 'express-validator' |
3 | import { exists, isBooleanValid, isIdOrUUIDValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc' | 3 | import { exists, isBooleanValid, isIdOrUUIDValid, isIdValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc' |
4 | import { logger } from '../../helpers/logger' | 4 | import { logger } from '../../helpers/logger' |
5 | import { areValidationErrors } from './utils' | 5 | import { areValidationErrors } from './utils' |
6 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' | 6 | import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' |
7 | import { isHostValid } from '../../helpers/custom-validators/servers' | 7 | import { isHostValid } from '../../helpers/custom-validators/servers' |
8 | import { ServerModel } from '../../models/server/server' | 8 | import { ServerModel } from '../../models/server/server' |
9 | import { doesVideoExist } from '../../helpers/middlewares' | 9 | import { doesVideoExist } from '../../helpers/middlewares' |
10 | import { isVideoRedundancyTarget } from '@server/helpers/custom-validators/video-redundancies' | ||
10 | 11 | ||
11 | const videoFileRedundancyGetValidator = [ | 12 | const videoFileRedundancyGetValidator = [ |
12 | param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), | 13 | param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'), |
@@ -101,10 +102,77 @@ const updateServerRedundancyValidator = [ | |||
101 | } | 102 | } |
102 | ] | 103 | ] |
103 | 104 | ||
105 | const listVideoRedundanciesValidator = [ | ||
106 | query('target') | ||
107 | .custom(isVideoRedundancyTarget).withMessage('Should have a valid video redundancies target'), | ||
108 | |||
109 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
110 | logger.debug('Checking listVideoRedundanciesValidator parameters', { parameters: req.query }) | ||
111 | |||
112 | if (areValidationErrors(req, res)) return | ||
113 | |||
114 | return next() | ||
115 | } | ||
116 | ] | ||
117 | |||
118 | const addVideoRedundancyValidator = [ | ||
119 | body('videoId') | ||
120 | .custom(isIdValid) | ||
121 | .withMessage('Should have a valid video id'), | ||
122 | |||
123 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
124 | logger.debug('Checking addVideoRedundancyValidator parameters', { parameters: req.query }) | ||
125 | |||
126 | if (areValidationErrors(req, res)) return | ||
127 | |||
128 | if (!await doesVideoExist(req.body.videoId, res, 'only-video')) return | ||
129 | |||
130 | if (res.locals.onlyVideo.remote === false) { | ||
131 | return res.status(400) | ||
132 | .json({ error: 'Cannot create a redundancy on a local video' }) | ||
133 | .end() | ||
134 | } | ||
135 | |||
136 | const alreadyExists = await VideoRedundancyModel.isLocalByVideoUUIDExists(res.locals.onlyVideo.uuid) | ||
137 | if (alreadyExists) { | ||
138 | return res.status(409) | ||
139 | .json({ error: 'This video is already duplicated by your instance.' }) | ||
140 | } | ||
141 | |||
142 | return next() | ||
143 | } | ||
144 | ] | ||
145 | |||
146 | const removeVideoRedundancyValidator = [ | ||
147 | param('redundancyId') | ||
148 | .custom(isIdValid) | ||
149 | .withMessage('Should have a valid redundancy id'), | ||
150 | |||
151 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
152 | logger.debug('Checking removeVideoRedundancyValidator parameters', { parameters: req.query }) | ||
153 | |||
154 | if (areValidationErrors(req, res)) return | ||
155 | |||
156 | const redundancy = await VideoRedundancyModel.loadByIdWithVideo(parseInt(req.params.redundancyId, 10)) | ||
157 | if (!redundancy) { | ||
158 | return res.status(404) | ||
159 | .json({ error: 'Video redundancy not found' }) | ||
160 | .end() | ||
161 | } | ||
162 | |||
163 | res.locals.videoRedundancy = redundancy | ||
164 | |||
165 | return next() | ||
166 | } | ||
167 | ] | ||
168 | |||
104 | // --------------------------------------------------------------------------- | 169 | // --------------------------------------------------------------------------- |
105 | 170 | ||
106 | export { | 171 | export { |
107 | videoFileRedundancyGetValidator, | 172 | videoFileRedundancyGetValidator, |
108 | videoPlaylistRedundancyGetValidator, | 173 | videoPlaylistRedundancyGetValidator, |
109 | updateServerRedundancyValidator | 174 | updateServerRedundancyValidator, |
175 | listVideoRedundanciesValidator, | ||
176 | addVideoRedundancyValidator, | ||
177 | removeVideoRedundancyValidator | ||
110 | } | 178 | } |
diff --git a/server/middlewares/validators/sort.ts b/server/middlewares/validators/sort.ts index c75e701d6..b76dab722 100644 --- a/server/middlewares/validators/sort.ts +++ b/server/middlewares/validators/sort.ts | |||
@@ -23,6 +23,7 @@ const SORTABLE_USER_NOTIFICATIONS_COLUMNS = createSortableColumns(SORTABLE_COLUM | |||
23 | const SORTABLE_VIDEO_PLAYLISTS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_PLAYLISTS) | 23 | const SORTABLE_VIDEO_PLAYLISTS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_PLAYLISTS) |
24 | const SORTABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.PLUGINS) | 24 | const SORTABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.PLUGINS) |
25 | const SORTABLE_AVAILABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.AVAILABLE_PLUGINS) | 25 | const SORTABLE_AVAILABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.AVAILABLE_PLUGINS) |
26 | const SORTABLE_VIDEO_REDUNDANCIES_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES) | ||
26 | 27 | ||
27 | const usersSortValidator = checkSort(SORTABLE_USERS_COLUMNS) | 28 | const usersSortValidator = checkSort(SORTABLE_USERS_COLUMNS) |
28 | const accountsSortValidator = checkSort(SORTABLE_ACCOUNTS_COLUMNS) | 29 | const accountsSortValidator = checkSort(SORTABLE_ACCOUNTS_COLUMNS) |
@@ -45,6 +46,7 @@ const userNotificationsSortValidator = checkSort(SORTABLE_USER_NOTIFICATIONS_COL | |||
45 | const videoPlaylistsSortValidator = checkSort(SORTABLE_VIDEO_PLAYLISTS_COLUMNS) | 46 | const videoPlaylistsSortValidator = checkSort(SORTABLE_VIDEO_PLAYLISTS_COLUMNS) |
46 | const pluginsSortValidator = checkSort(SORTABLE_PLUGINS_COLUMNS) | 47 | const pluginsSortValidator = checkSort(SORTABLE_PLUGINS_COLUMNS) |
47 | const availablePluginsSortValidator = checkSort(SORTABLE_AVAILABLE_PLUGINS_COLUMNS) | 48 | const availablePluginsSortValidator = checkSort(SORTABLE_AVAILABLE_PLUGINS_COLUMNS) |
49 | const videoRedundanciesSortValidator = checkSort(SORTABLE_VIDEO_REDUNDANCIES_COLUMNS) | ||
48 | 50 | ||
49 | // --------------------------------------------------------------------------- | 51 | // --------------------------------------------------------------------------- |
50 | 52 | ||
@@ -69,5 +71,6 @@ export { | |||
69 | serversBlocklistSortValidator, | 71 | serversBlocklistSortValidator, |
70 | userNotificationsSortValidator, | 72 | userNotificationsSortValidator, |
71 | videoPlaylistsSortValidator, | 73 | videoPlaylistsSortValidator, |
74 | videoRedundanciesSortValidator, | ||
72 | pluginsSortValidator | 75 | pluginsSortValidator |
73 | } | 76 | } |
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index 8c9a7eabf..4e66d72e3 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts | |||
@@ -13,13 +13,13 @@ import { | |||
13 | UpdatedAt | 13 | UpdatedAt |
14 | } from 'sequelize-typescript' | 14 | } from 'sequelize-typescript' |
15 | import { ActorModel } from '../activitypub/actor' | 15 | import { ActorModel } from '../activitypub/actor' |
16 | import { getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils' | 16 | import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils' |
17 | import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc' | 17 | import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc' |
18 | import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants' | 18 | import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants' |
19 | import { VideoFileModel } from '../video/video-file' | 19 | import { VideoFileModel } from '../video/video-file' |
20 | import { getServerActor } from '../../helpers/utils' | 20 | import { getServerActor } from '../../helpers/utils' |
21 | import { VideoModel } from '../video/video' | 21 | import { VideoModel } from '../video/video' |
22 | import { VideoRedundancyStrategy } from '../../../shared/models/redundancy' | 22 | import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy' |
23 | import { logger } from '../../helpers/logger' | 23 | import { logger } from '../../helpers/logger' |
24 | import { CacheFileObject, VideoPrivacy } from '../../../shared' | 24 | import { CacheFileObject, VideoPrivacy } from '../../../shared' |
25 | import { VideoChannelModel } from '../video/video-channel' | 25 | import { VideoChannelModel } from '../video/video-channel' |
@@ -27,10 +27,16 @@ import { ServerModel } from '../server/server' | |||
27 | import { sample } from 'lodash' | 27 | import { sample } from 'lodash' |
28 | import { isTestInstance } from '../../helpers/core-utils' | 28 | import { isTestInstance } from '../../helpers/core-utils' |
29 | import * as Bluebird from 'bluebird' | 29 | import * as Bluebird from 'bluebird' |
30 | import { col, FindOptions, fn, literal, Op, Transaction } from 'sequelize' | 30 | import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize' |
31 | import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist' | 31 | import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist' |
32 | import { CONFIG } from '../../initializers/config' | 32 | import { CONFIG } from '../../initializers/config' |
33 | import { MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/typings/models' | 33 | import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/typings/models' |
34 | import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model' | ||
35 | import { | ||
36 | FileRedundancyInformation, | ||
37 | StreamingPlaylistRedundancyInformation, | ||
38 | VideoRedundancy | ||
39 | } from '@shared/models/redundancy/video-redundancy.model' | ||
34 | 40 | ||
35 | export enum ScopeNames { | 41 | export enum ScopeNames { |
36 | WITH_VIDEO = 'WITH_VIDEO' | 42 | WITH_VIDEO = 'WITH_VIDEO' |
@@ -86,7 +92,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
86 | @UpdatedAt | 92 | @UpdatedAt |
87 | updatedAt: Date | 93 | updatedAt: Date |
88 | 94 | ||
89 | @AllowNull(false) | 95 | @AllowNull(true) |
90 | @Column | 96 | @Column |
91 | expiresOn: Date | 97 | expiresOn: Date |
92 | 98 | ||
@@ -193,6 +199,15 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
193 | return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) | 199 | return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) |
194 | } | 200 | } |
195 | 201 | ||
202 | static loadByIdWithVideo (id: number, transaction?: Transaction): Bluebird<MVideoRedundancyVideo> { | ||
203 | const query = { | ||
204 | where: { id }, | ||
205 | transaction | ||
206 | } | ||
207 | |||
208 | return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) | ||
209 | } | ||
210 | |||
196 | static loadByUrl (url: string, transaction?: Transaction): Bluebird<MVideoRedundancy> { | 211 | static loadByUrl (url: string, transaction?: Transaction): Bluebird<MVideoRedundancy> { |
197 | const query = { | 212 | const query = { |
198 | where: { | 213 | where: { |
@@ -394,7 +409,8 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
394 | [Op.ne]: actor.id | 409 | [Op.ne]: actor.id |
395 | }, | 410 | }, |
396 | expiresOn: { | 411 | expiresOn: { |
397 | [ Op.lt ]: new Date() | 412 | [ Op.lt ]: new Date(), |
413 | [ Op.ne ]: null | ||
398 | } | 414 | } |
399 | } | 415 | } |
400 | } | 416 | } |
@@ -447,7 +463,112 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
447 | return VideoRedundancyModel.findAll(query) | 463 | return VideoRedundancyModel.findAll(query) |
448 | } | 464 | } |
449 | 465 | ||
450 | static async getStats (strategy: VideoRedundancyStrategy) { | 466 | static listForApi (options: { |
467 | start: number, | ||
468 | count: number, | ||
469 | sort: string, | ||
470 | target: VideoRedundanciesTarget, | ||
471 | strategy?: string | ||
472 | }) { | ||
473 | const { start, count, sort, target, strategy } = options | ||
474 | let redundancyWhere: WhereOptions = {} | ||
475 | let videosWhere: WhereOptions = {} | ||
476 | let redundancySqlSuffix = '' | ||
477 | |||
478 | if (target === 'my-videos') { | ||
479 | Object.assign(videosWhere, { remote: false }) | ||
480 | } else if (target === 'remote-videos') { | ||
481 | Object.assign(videosWhere, { remote: true }) | ||
482 | Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } }) | ||
483 | redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL' | ||
484 | } | ||
485 | |||
486 | if (strategy) { | ||
487 | Object.assign(redundancyWhere, { strategy: strategy }) | ||
488 | } | ||
489 | |||
490 | const videoFilterWhere = { | ||
491 | [Op.and]: [ | ||
492 | { | ||
493 | [ Op.or ]: [ | ||
494 | { | ||
495 | id: { | ||
496 | [ Op.in ]: literal( | ||
497 | '(' + | ||
498 | 'SELECT "videoId" FROM "videoFile" ' + | ||
499 | 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' + | ||
500 | redundancySqlSuffix + | ||
501 | ')' | ||
502 | ) | ||
503 | } | ||
504 | }, | ||
505 | { | ||
506 | id: { | ||
507 | [ Op.in ]: literal( | ||
508 | '(' + | ||
509 | 'select "videoId" FROM "videoStreamingPlaylist" ' + | ||
510 | 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' + | ||
511 | redundancySqlSuffix + | ||
512 | ')' | ||
513 | ) | ||
514 | } | ||
515 | } | ||
516 | ] | ||
517 | }, | ||
518 | |||
519 | videosWhere | ||
520 | ] | ||
521 | } | ||
522 | |||
523 | // /!\ On video model /!\ | ||
524 | const findOptions = { | ||
525 | offset: start, | ||
526 | limit: count, | ||
527 | order: getSort(sort), | ||
528 | include: [ | ||
529 | { | ||
530 | required: false, | ||
531 | model: VideoFileModel.unscoped(), | ||
532 | include: [ | ||
533 | { | ||
534 | model: VideoRedundancyModel.unscoped(), | ||
535 | required: false, | ||
536 | where: redundancyWhere | ||
537 | } | ||
538 | ] | ||
539 | }, | ||
540 | { | ||
541 | required: false, | ||
542 | model: VideoStreamingPlaylistModel.unscoped(), | ||
543 | include: [ | ||
544 | { | ||
545 | model: VideoRedundancyModel.unscoped(), | ||
546 | required: false, | ||
547 | where: redundancyWhere | ||
548 | }, | ||
549 | { | ||
550 | model: VideoFileModel.unscoped(), | ||
551 | required: false | ||
552 | } | ||
553 | ] | ||
554 | } | ||
555 | ], | ||
556 | where: videoFilterWhere | ||
557 | } | ||
558 | |||
559 | // /!\ On video model /!\ | ||
560 | const countOptions = { | ||
561 | where: videoFilterWhere | ||
562 | } | ||
563 | |||
564 | return Promise.all([ | ||
565 | VideoModel.findAll(findOptions), | ||
566 | |||
567 | VideoModel.count(countOptions) | ||
568 | ]).then(([ data, total ]) => ({ total, data })) | ||
569 | } | ||
570 | |||
571 | static async getStats (strategy: VideoRedundancyStrategyWithManual) { | ||
451 | const actor = await getServerActor() | 572 | const actor = await getServerActor() |
452 | 573 | ||
453 | const query: FindOptions = { | 574 | const query: FindOptions = { |
@@ -478,6 +599,53 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
478 | })) | 599 | })) |
479 | } | 600 | } |
480 | 601 | ||
602 | static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy { | ||
603 | let filesRedundancies: FileRedundancyInformation[] = [] | ||
604 | let streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = [] | ||
605 | |||
606 | for (const file of video.VideoFiles) { | ||
607 | for (const redundancy of file.RedundancyVideos) { | ||
608 | filesRedundancies.push({ | ||
609 | id: redundancy.id, | ||
610 | fileUrl: redundancy.fileUrl, | ||
611 | strategy: redundancy.strategy, | ||
612 | createdAt: redundancy.createdAt, | ||
613 | updatedAt: redundancy.updatedAt, | ||
614 | expiresOn: redundancy.expiresOn, | ||
615 | size: file.size | ||
616 | }) | ||
617 | } | ||
618 | } | ||
619 | |||
620 | for (const playlist of video.VideoStreamingPlaylists) { | ||
621 | const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0) | ||
622 | |||
623 | for (const redundancy of playlist.RedundancyVideos) { | ||
624 | streamingPlaylistsRedundancies.push({ | ||
625 | id: redundancy.id, | ||
626 | fileUrl: redundancy.fileUrl, | ||
627 | strategy: redundancy.strategy, | ||
628 | createdAt: redundancy.createdAt, | ||
629 | updatedAt: redundancy.updatedAt, | ||
630 | expiresOn: redundancy.expiresOn, | ||
631 | size | ||
632 | }) | ||
633 | } | ||
634 | } | ||
635 | |||
636 | return { | ||
637 | id: video.id, | ||
638 | name: video.name, | ||
639 | url: video.url, | ||
640 | uuid: video.uuid, | ||
641 | |||
642 | redundancies: { | ||
643 | files: filesRedundancies, | ||
644 | streamingPlaylists: streamingPlaylistsRedundancies | ||
645 | } | ||
646 | } | ||
647 | } | ||
648 | |||
481 | getVideo () { | 649 | getVideo () { |
482 | if (this.VideoFile) return this.VideoFile.Video | 650 | if (this.VideoFile) return this.VideoFile.Video |
483 | 651 | ||
@@ -494,7 +662,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
494 | id: this.url, | 662 | id: this.url, |
495 | type: 'CacheFile' as 'CacheFile', | 663 | type: 'CacheFile' as 'CacheFile', |
496 | object: this.VideoStreamingPlaylist.Video.url, | 664 | object: this.VideoStreamingPlaylist.Video.url, |
497 | expires: this.expiresOn.toISOString(), | 665 | expires: this.expiresOn ? this.expiresOn.toISOString() : null, |
498 | url: { | 666 | url: { |
499 | type: 'Link', | 667 | type: 'Link', |
500 | mediaType: 'application/x-mpegURL', | 668 | mediaType: 'application/x-mpegURL', |
@@ -507,7 +675,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> { | |||
507 | id: this.url, | 675 | id: this.url, |
508 | type: 'CacheFile' as 'CacheFile', | 676 | type: 'CacheFile' as 'CacheFile', |
509 | object: this.VideoFile.Video.url, | 677 | object: this.VideoFile.Video.url, |
510 | expires: this.expiresOn.toISOString(), | 678 | expires: this.expiresOn ? this.expiresOn.toISOString() : null, |
511 | url: { | 679 | url: { |
512 | type: 'Link', | 680 | type: 'Link', |
513 | mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any, | 681 | mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any, |
diff --git a/server/tests/api/check-params/redundancy.ts b/server/tests/api/check-params/redundancy.ts index 6471da840..7012a39ee 100644 --- a/server/tests/api/check-params/redundancy.ts +++ b/server/tests/api/check-params/redundancy.ts | |||
@@ -3,21 +3,25 @@ | |||
3 | import 'mocha' | 3 | import 'mocha' |
4 | 4 | ||
5 | import { | 5 | import { |
6 | checkBadCountPagination, | ||
7 | checkBadSortPagination, | ||
8 | checkBadStartPagination, | ||
6 | cleanupTests, | 9 | cleanupTests, |
7 | createUser, | 10 | createUser, |
8 | doubleFollow, | 11 | doubleFollow, |
9 | flushAndRunMultipleServers, | 12 | flushAndRunMultipleServers, makeDeleteRequest, |
10 | flushTests, | 13 | makeGetRequest, makePostBodyRequest, |
11 | killallServers, | ||
12 | makePutBodyRequest, | 14 | makePutBodyRequest, |
13 | ServerInfo, | 15 | ServerInfo, |
14 | setAccessTokensToServers, | 16 | setAccessTokensToServers, uploadVideoAndGetId, |
15 | userLogin | 17 | userLogin, waitJobs |
16 | } from '../../../../shared/extra-utils' | 18 | } from '../../../../shared/extra-utils' |
17 | 19 | ||
18 | describe('Test server redundancy API validators', function () { | 20 | describe('Test server redundancy API validators', function () { |
19 | let servers: ServerInfo[] | 21 | let servers: ServerInfo[] |
20 | let userAccessToken = null | 22 | let userAccessToken = null |
23 | let videoIdLocal: number | ||
24 | let videoIdRemote: number | ||
21 | 25 | ||
22 | // --------------------------------------------------------------- | 26 | // --------------------------------------------------------------- |
23 | 27 | ||
@@ -36,9 +40,134 @@ describe('Test server redundancy API validators', function () { | |||
36 | 40 | ||
37 | await createUser({ url: servers[ 0 ].url, accessToken: servers[ 0 ].accessToken, username: user.username, password: user.password }) | 41 | await createUser({ url: servers[ 0 ].url, accessToken: servers[ 0 ].accessToken, username: user.username, password: user.password }) |
38 | userAccessToken = await userLogin(servers[0], user) | 42 | userAccessToken = await userLogin(servers[0], user) |
43 | |||
44 | videoIdLocal = (await uploadVideoAndGetId({ server: servers[0], videoName: 'video' })).id | ||
45 | videoIdRemote = (await uploadVideoAndGetId({ server: servers[1], videoName: 'video' })).id | ||
46 | |||
47 | await waitJobs(servers) | ||
48 | }) | ||
49 | |||
50 | describe('When listing redundancies', function () { | ||
51 | const path = '/api/v1/server/redundancy/videos' | ||
52 | |||
53 | let url: string | ||
54 | let token: string | ||
55 | |||
56 | before(function () { | ||
57 | url = servers[0].url | ||
58 | token = servers[0].accessToken | ||
59 | }) | ||
60 | |||
61 | it('Should fail with an invalid token', async function () { | ||
62 | await makeGetRequest({ url, path, token: 'fake_token', statusCodeExpected: 401 }) | ||
63 | }) | ||
64 | |||
65 | it('Should fail if the user is not an administrator', async function () { | ||
66 | await makeGetRequest({ url, path, token: userAccessToken, statusCodeExpected: 403 }) | ||
67 | }) | ||
68 | |||
69 | it('Should fail with a bad start pagination', async function () { | ||
70 | await checkBadStartPagination(url, path, servers[0].accessToken) | ||
71 | }) | ||
72 | |||
73 | it('Should fail with a bad count pagination', async function () { | ||
74 | await checkBadCountPagination(url, path, servers[0].accessToken) | ||
75 | }) | ||
76 | |||
77 | it('Should fail with an incorrect sort', async function () { | ||
78 | await checkBadSortPagination(url, path, servers[0].accessToken) | ||
79 | }) | ||
80 | |||
81 | it('Should fail with a bad target', async function () { | ||
82 | await makeGetRequest({ url, path, token, query: { target: 'bad target' } }) | ||
83 | }) | ||
84 | |||
85 | it('Should fail without target', async function () { | ||
86 | await makeGetRequest({ url, path, token }) | ||
87 | }) | ||
88 | |||
89 | it('Should succeed with the correct params', async function () { | ||
90 | await makeGetRequest({ url, path, token, query: { target: 'my-videos' }, statusCodeExpected: 200 }) | ||
91 | }) | ||
92 | }) | ||
93 | |||
94 | describe('When manually adding a redundancy', function () { | ||
95 | const path = '/api/v1/server/redundancy/videos' | ||
96 | |||
97 | let url: string | ||
98 | let token: string | ||
99 | |||
100 | before(function () { | ||
101 | url = servers[0].url | ||
102 | token = servers[0].accessToken | ||
103 | }) | ||
104 | |||
105 | it('Should fail with an invalid token', async function () { | ||
106 | await makePostBodyRequest({ url, path, token: 'fake_token', statusCodeExpected: 401 }) | ||
107 | }) | ||
108 | |||
109 | it('Should fail if the user is not an administrator', async function () { | ||
110 | await makePostBodyRequest({ url, path, token: userAccessToken, statusCodeExpected: 403 }) | ||
111 | }) | ||
112 | |||
113 | it('Should fail without a video id', async function () { | ||
114 | await makePostBodyRequest({ url, path, token }) | ||
115 | }) | ||
116 | |||
117 | it('Should fail with an incorrect video id', async function () { | ||
118 | await makePostBodyRequest({ url, path, token, fields: { videoId: 'peertube' } }) | ||
119 | }) | ||
120 | |||
121 | it('Should fail with a not found video id', async function () { | ||
122 | await makePostBodyRequest({ url, path, token, fields: { videoId: 6565 }, statusCodeExpected: 404 }) | ||
123 | }) | ||
124 | |||
125 | it('Should fail with a local a video id', async function () { | ||
126 | await makePostBodyRequest({ url, path, token, fields: { videoId: videoIdLocal } }) | ||
127 | }) | ||
128 | |||
129 | it('Should succeed with the correct params', async function () { | ||
130 | await makePostBodyRequest({ url, path, token, fields: { videoId: videoIdRemote }, statusCodeExpected: 204 }) | ||
131 | }) | ||
132 | |||
133 | it('Should fail if the video is already duplicated', async function () { | ||
134 | this.timeout(30000) | ||
135 | |||
136 | await waitJobs(servers) | ||
137 | |||
138 | await makePostBodyRequest({ url, path, token, fields: { videoId: videoIdRemote }, statusCodeExpected: 409 }) | ||
139 | }) | ||
140 | }) | ||
141 | |||
142 | describe('When manually removing a redundancy', function () { | ||
143 | const path = '/api/v1/server/redundancy/videos/' | ||
144 | |||
145 | let url: string | ||
146 | let token: string | ||
147 | |||
148 | before(function () { | ||
149 | url = servers[0].url | ||
150 | token = servers[0].accessToken | ||
151 | }) | ||
152 | |||
153 | it('Should fail with an invalid token', async function () { | ||
154 | await makeDeleteRequest({ url, path: path + '1', token: 'fake_token', statusCodeExpected: 401 }) | ||
155 | }) | ||
156 | |||
157 | it('Should fail if the user is not an administrator', async function () { | ||
158 | await makeDeleteRequest({ url, path: path + '1', token: userAccessToken, statusCodeExpected: 403 }) | ||
159 | }) | ||
160 | |||
161 | it('Should fail with an incorrect video id', async function () { | ||
162 | await makeDeleteRequest({ url, path: path + 'toto', token }) | ||
163 | }) | ||
164 | |||
165 | it('Should fail with a not found video redundancy', async function () { | ||
166 | await makeDeleteRequest({ url, path: path + '454545', token, statusCodeExpected: 404 }) | ||
167 | }) | ||
39 | }) | 168 | }) |
40 | 169 | ||
41 | describe('When updating redundancy', function () { | 170 | describe('When updating server redundancy', function () { |
42 | const path = '/api/v1/server/redundancy' | 171 | const path = '/api/v1/server/redundancy' |
43 | 172 | ||
44 | it('Should fail with an invalid token', async function () { | 173 | it('Should fail with an invalid token', async function () { |
diff --git a/server/tests/api/redundancy/index.ts b/server/tests/api/redundancy/index.ts index 8e69b95a6..5359055b0 100644 --- a/server/tests/api/redundancy/index.ts +++ b/server/tests/api/redundancy/index.ts | |||
@@ -1 +1,2 @@ | |||
1 | import './redundancy' | 1 | import './redundancy' |
2 | import './manage-redundancy' | ||
diff --git a/server/tests/api/redundancy/manage-redundancy.ts b/server/tests/api/redundancy/manage-redundancy.ts new file mode 100644 index 000000000..6a8937f24 --- /dev/null +++ b/server/tests/api/redundancy/manage-redundancy.ts | |||
@@ -0,0 +1,373 @@ | |||
1 | /* tslint:disable:no-unused-expression */ | ||
2 | |||
3 | import * as chai from 'chai' | ||
4 | import 'mocha' | ||
5 | import { | ||
6 | cleanupTests, | ||
7 | doubleFollow, | ||
8 | flushAndRunMultipleServers, | ||
9 | getLocalIdByUUID, | ||
10 | ServerInfo, | ||
11 | setAccessTokensToServers, | ||
12 | uploadVideo, | ||
13 | uploadVideoAndGetId, | ||
14 | waitUntilLog | ||
15 | } from '../../../../shared/extra-utils' | ||
16 | import { waitJobs } from '../../../../shared/extra-utils/server/jobs' | ||
17 | import { addVideoRedundancy, listVideoRedundancies, removeVideoRedundancy, updateRedundancy } from '@shared/extra-utils/server/redundancy' | ||
18 | import { VideoPrivacy, VideoRedundanciesTarget, VideoRedundancy } from '@shared/models' | ||
19 | |||
20 | const expect = chai.expect | ||
21 | |||
22 | describe('Test manage videos redundancy', function () { | ||
23 | const targets: VideoRedundanciesTarget[] = [ 'my-videos', 'remote-videos' ] | ||
24 | |||
25 | let servers: ServerInfo[] | ||
26 | let video1Server2UUID: string | ||
27 | let video2Server2UUID: string | ||
28 | let redundanciesToRemove: number[] = [] | ||
29 | |||
30 | before(async function () { | ||
31 | this.timeout(120000) | ||
32 | |||
33 | const config = { | ||
34 | transcoding: { | ||
35 | hls: { | ||
36 | enabled: true | ||
37 | } | ||
38 | }, | ||
39 | redundancy: { | ||
40 | videos: { | ||
41 | check_interval: '1 second', | ||
42 | strategies: [ | ||
43 | { | ||
44 | strategy: 'recently-added', | ||
45 | min_lifetime: '1 hour', | ||
46 | size: '10MB', | ||
47 | min_views: 0 | ||
48 | } | ||
49 | ] | ||
50 | } | ||
51 | } | ||
52 | } | ||
53 | servers = await flushAndRunMultipleServers(3, config) | ||
54 | |||
55 | // Get the access tokens | ||
56 | await setAccessTokensToServers(servers) | ||
57 | |||
58 | { | ||
59 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) | ||
60 | video1Server2UUID = res.body.video.uuid | ||
61 | } | ||
62 | |||
63 | { | ||
64 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) | ||
65 | video2Server2UUID = res.body.video.uuid | ||
66 | } | ||
67 | |||
68 | await waitJobs(servers) | ||
69 | |||
70 | // Server 1 and server 2 follow each other | ||
71 | await doubleFollow(servers[ 0 ], servers[ 1 ]) | ||
72 | await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) | ||
73 | |||
74 | await waitJobs(servers) | ||
75 | }) | ||
76 | |||
77 | it('Should not have redundancies on server 3', async function () { | ||
78 | for (const target of targets) { | ||
79 | const res = await listVideoRedundancies({ | ||
80 | url: servers[2].url, | ||
81 | accessToken: servers[2].accessToken, | ||
82 | target | ||
83 | }) | ||
84 | |||
85 | expect(res.body.total).to.equal(0) | ||
86 | expect(res.body.data).to.have.lengthOf(0) | ||
87 | } | ||
88 | }) | ||
89 | |||
90 | it('Should not have "remote-videos" redundancies on server 2', async function () { | ||
91 | this.timeout(120000) | ||
92 | |||
93 | await waitJobs(servers) | ||
94 | await waitUntilLog(servers[0], 'Duplicated ', 10) | ||
95 | await waitJobs(servers) | ||
96 | |||
97 | const res = await listVideoRedundancies({ | ||
98 | url: servers[1].url, | ||
99 | accessToken: servers[1].accessToken, | ||
100 | target: 'remote-videos' | ||
101 | }) | ||
102 | |||
103 | expect(res.body.total).to.equal(0) | ||
104 | expect(res.body.data).to.have.lengthOf(0) | ||
105 | }) | ||
106 | |||
107 | it('Should have "my-videos" redundancies on server 2', async function () { | ||
108 | this.timeout(120000) | ||
109 | |||
110 | const res = await listVideoRedundancies({ | ||
111 | url: servers[1].url, | ||
112 | accessToken: servers[1].accessToken, | ||
113 | target: 'my-videos' | ||
114 | }) | ||
115 | |||
116 | expect(res.body.total).to.equal(2) | ||
117 | |||
118 | const videos = res.body.data as VideoRedundancy[] | ||
119 | expect(videos).to.have.lengthOf(2) | ||
120 | |||
121 | const videos1 = videos.find(v => v.uuid === video1Server2UUID) | ||
122 | const videos2 = videos.find(v => v.uuid === video2Server2UUID) | ||
123 | |||
124 | expect(videos1.name).to.equal('video 1 server 2') | ||
125 | expect(videos2.name).to.equal('video 2 server 2') | ||
126 | |||
127 | expect(videos1.redundancies.files).to.have.lengthOf(4) | ||
128 | expect(videos1.redundancies.streamingPlaylists).to.have.lengthOf(1) | ||
129 | |||
130 | const redundancies = videos1.redundancies.files.concat(videos1.redundancies.streamingPlaylists) | ||
131 | |||
132 | for (const r of redundancies) { | ||
133 | expect(r.strategy).to.be.null | ||
134 | expect(r.fileUrl).to.exist | ||
135 | expect(r.createdAt).to.exist | ||
136 | expect(r.updatedAt).to.exist | ||
137 | expect(r.expiresOn).to.exist | ||
138 | } | ||
139 | }) | ||
140 | |||
141 | it('Should not have "my-videos" redundancies on server 1', async function () { | ||
142 | const res = await listVideoRedundancies({ | ||
143 | url: servers[0].url, | ||
144 | accessToken: servers[0].accessToken, | ||
145 | target: 'my-videos' | ||
146 | }) | ||
147 | |||
148 | expect(res.body.total).to.equal(0) | ||
149 | expect(res.body.data).to.have.lengthOf(0) | ||
150 | }) | ||
151 | |||
152 | it('Should have "remote-videos" redundancies on server 1', async function () { | ||
153 | this.timeout(120000) | ||
154 | |||
155 | const res = await listVideoRedundancies({ | ||
156 | url: servers[0].url, | ||
157 | accessToken: servers[0].accessToken, | ||
158 | target: 'remote-videos' | ||
159 | }) | ||
160 | |||
161 | expect(res.body.total).to.equal(2) | ||
162 | |||
163 | const videos = res.body.data as VideoRedundancy[] | ||
164 | expect(videos).to.have.lengthOf(2) | ||
165 | |||
166 | const videos1 = videos.find(v => v.uuid === video1Server2UUID) | ||
167 | const videos2 = videos.find(v => v.uuid === video2Server2UUID) | ||
168 | |||
169 | expect(videos1.name).to.equal('video 1 server 2') | ||
170 | expect(videos2.name).to.equal('video 2 server 2') | ||
171 | |||
172 | expect(videos1.redundancies.files).to.have.lengthOf(4) | ||
173 | expect(videos1.redundancies.streamingPlaylists).to.have.lengthOf(1) | ||
174 | |||
175 | const redundancies = videos1.redundancies.files.concat(videos1.redundancies.streamingPlaylists) | ||
176 | |||
177 | for (const r of redundancies) { | ||
178 | expect(r.strategy).to.equal('recently-added') | ||
179 | expect(r.fileUrl).to.exist | ||
180 | expect(r.createdAt).to.exist | ||
181 | expect(r.updatedAt).to.exist | ||
182 | expect(r.expiresOn).to.exist | ||
183 | } | ||
184 | }) | ||
185 | |||
186 | it('Should correctly paginate and sort results', async function () { | ||
187 | { | ||
188 | const res = await listVideoRedundancies({ | ||
189 | url: servers[0].url, | ||
190 | accessToken: servers[0].accessToken, | ||
191 | target: 'remote-videos', | ||
192 | sort: 'name', | ||
193 | start: 0, | ||
194 | count: 2 | ||
195 | }) | ||
196 | |||
197 | const videos = res.body.data | ||
198 | expect(videos[ 0 ].name).to.equal('video 1 server 2') | ||
199 | expect(videos[ 1 ].name).to.equal('video 2 server 2') | ||
200 | } | ||
201 | |||
202 | { | ||
203 | const res = await listVideoRedundancies({ | ||
204 | url: servers[0].url, | ||
205 | accessToken: servers[0].accessToken, | ||
206 | target: 'remote-videos', | ||
207 | sort: '-name', | ||
208 | start: 0, | ||
209 | count: 2 | ||
210 | }) | ||
211 | |||
212 | const videos = res.body.data | ||
213 | expect(videos[ 0 ].name).to.equal('video 2 server 2') | ||
214 | expect(videos[ 1 ].name).to.equal('video 1 server 2') | ||
215 | } | ||
216 | |||
217 | { | ||
218 | const res = await listVideoRedundancies({ | ||
219 | url: servers[0].url, | ||
220 | accessToken: servers[0].accessToken, | ||
221 | target: 'remote-videos', | ||
222 | sort: '-name', | ||
223 | start: 1, | ||
224 | count: 1 | ||
225 | }) | ||
226 | |||
227 | const videos = res.body.data | ||
228 | expect(videos[ 0 ].name).to.equal('video 1 server 2') | ||
229 | } | ||
230 | }) | ||
231 | |||
232 | it('Should manually add a redundancy and list it', async function () { | ||
233 | this.timeout(120000) | ||
234 | |||
235 | const uuid = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video 3 server 2', privacy: VideoPrivacy.UNLISTED })).uuid | ||
236 | await waitJobs(servers) | ||
237 | const videoId = await getLocalIdByUUID(servers[0].url, uuid) | ||
238 | |||
239 | await addVideoRedundancy({ | ||
240 | url: servers[0].url, | ||
241 | accessToken: servers[0].accessToken, | ||
242 | videoId | ||
243 | }) | ||
244 | |||
245 | await waitJobs(servers) | ||
246 | await waitUntilLog(servers[0], 'Duplicated ', 15) | ||
247 | await waitJobs(servers) | ||
248 | |||
249 | { | ||
250 | const res = await listVideoRedundancies({ | ||
251 | url: servers[0].url, | ||
252 | accessToken: servers[0].accessToken, | ||
253 | target: 'remote-videos', | ||
254 | sort: '-name', | ||
255 | start: 0, | ||
256 | count: 5 | ||
257 | }) | ||
258 | |||
259 | const videos = res.body.data | ||
260 | expect(videos[ 0 ].name).to.equal('video 3 server 2') | ||
261 | |||
262 | const video = videos[ 0 ] | ||
263 | expect(video.redundancies.files).to.have.lengthOf(4) | ||
264 | expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) | ||
265 | |||
266 | const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists) | ||
267 | |||
268 | for (const r of redundancies) { | ||
269 | redundanciesToRemove.push(r.id) | ||
270 | |||
271 | expect(r.strategy).to.equal('manual') | ||
272 | expect(r.fileUrl).to.exist | ||
273 | expect(r.createdAt).to.exist | ||
274 | expect(r.updatedAt).to.exist | ||
275 | expect(r.expiresOn).to.be.null | ||
276 | } | ||
277 | } | ||
278 | |||
279 | const res = await listVideoRedundancies({ | ||
280 | url: servers[1].url, | ||
281 | accessToken: servers[1].accessToken, | ||
282 | target: 'my-videos', | ||
283 | sort: '-name', | ||
284 | start: 0, | ||
285 | count: 5 | ||
286 | }) | ||
287 | |||
288 | const videos = res.body.data | ||
289 | expect(videos[ 0 ].name).to.equal('video 3 server 2') | ||
290 | |||
291 | const video = videos[ 0 ] | ||
292 | expect(video.redundancies.files).to.have.lengthOf(4) | ||
293 | expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) | ||
294 | |||
295 | const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists) | ||
296 | |||
297 | for (const r of redundancies) { | ||
298 | expect(r.strategy).to.be.null | ||
299 | expect(r.fileUrl).to.exist | ||
300 | expect(r.createdAt).to.exist | ||
301 | expect(r.updatedAt).to.exist | ||
302 | expect(r.expiresOn).to.be.null | ||
303 | } | ||
304 | }) | ||
305 | |||
306 | it('Should manually remove a redundancy and remove it from the list', async function () { | ||
307 | this.timeout(120000) | ||
308 | |||
309 | for (const redundancyId of redundanciesToRemove) { | ||
310 | await removeVideoRedundancy({ | ||
311 | url: servers[ 0 ].url, | ||
312 | accessToken: servers[ 0 ].accessToken, | ||
313 | redundancyId | ||
314 | }) | ||
315 | } | ||
316 | |||
317 | { | ||
318 | const res = await listVideoRedundancies({ | ||
319 | url: servers[0].url, | ||
320 | accessToken: servers[0].accessToken, | ||
321 | target: 'remote-videos', | ||
322 | sort: '-name', | ||
323 | start: 0, | ||
324 | count: 5 | ||
325 | }) | ||
326 | |||
327 | const videos = res.body.data | ||
328 | expect(videos).to.have.lengthOf(2) | ||
329 | |||
330 | expect(videos[ 0 ].name).to.equal('video 2 server 2') | ||
331 | |||
332 | redundanciesToRemove = [] | ||
333 | const video = videos[ 0 ] | ||
334 | expect(video.redundancies.files).to.have.lengthOf(4) | ||
335 | expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) | ||
336 | |||
337 | const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists) | ||
338 | |||
339 | for (const r of redundancies) { | ||
340 | redundanciesToRemove.push(r.id) | ||
341 | } | ||
342 | } | ||
343 | }) | ||
344 | |||
345 | it('Should remove another (auto) redundancy', async function () { | ||
346 | { | ||
347 | for (const redundancyId of redundanciesToRemove) { | ||
348 | await removeVideoRedundancy({ | ||
349 | url: servers[ 0 ].url, | ||
350 | accessToken: servers[ 0 ].accessToken, | ||
351 | redundancyId | ||
352 | }) | ||
353 | } | ||
354 | |||
355 | const res = await listVideoRedundancies({ | ||
356 | url: servers[0].url, | ||
357 | accessToken: servers[0].accessToken, | ||
358 | target: 'remote-videos', | ||
359 | sort: '-name', | ||
360 | start: 0, | ||
361 | count: 5 | ||
362 | }) | ||
363 | |||
364 | const videos = res.body.data | ||
365 | expect(videos[ 0 ].name).to.equal('video 1 server 2') | ||
366 | expect(videos).to.have.lengthOf(1) | ||
367 | } | ||
368 | }) | ||
369 | |||
370 | after(async function () { | ||
371 | await cleanupTests(servers) | ||
372 | }) | ||
373 | }) | ||
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index 1cdf93aa1..f5bf130d5 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts | |||
@@ -5,7 +5,8 @@ import 'mocha' | |||
5 | import { VideoDetails } from '../../../../shared/models/videos' | 5 | import { VideoDetails } from '../../../../shared/models/videos' |
6 | import { | 6 | import { |
7 | checkSegmentHash, | 7 | checkSegmentHash, |
8 | checkVideoFilesWereRemoved, cleanupTests, | 8 | checkVideoFilesWereRemoved, |
9 | cleanupTests, | ||
9 | doubleFollow, | 10 | doubleFollow, |
10 | flushAndRunMultipleServers, | 11 | flushAndRunMultipleServers, |
11 | getFollowingListPaginationAndSort, | 12 | getFollowingListPaginationAndSort, |
@@ -28,11 +29,16 @@ import { | |||
28 | import { waitJobs } from '../../../../shared/extra-utils/server/jobs' | 29 | import { waitJobs } from '../../../../shared/extra-utils/server/jobs' |
29 | 30 | ||
30 | import * as magnetUtil from 'magnet-uri' | 31 | import * as magnetUtil from 'magnet-uri' |
31 | import { updateRedundancy } from '../../../../shared/extra-utils/server/redundancy' | 32 | import { |
33 | addVideoRedundancy, | ||
34 | listVideoRedundancies, | ||
35 | removeVideoRedundancy, | ||
36 | updateRedundancy | ||
37 | } from '../../../../shared/extra-utils/server/redundancy' | ||
32 | import { ActorFollow } from '../../../../shared/models/actors' | 38 | import { ActorFollow } from '../../../../shared/models/actors' |
33 | import { readdir } from 'fs-extra' | 39 | import { readdir } from 'fs-extra' |
34 | import { join } from 'path' | 40 | import { join } from 'path' |
35 | import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy' | 41 | import { VideoRedundancy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../../shared/models/redundancy' |
36 | import { getStats } from '../../../../shared/extra-utils/server/stats' | 42 | import { getStats } from '../../../../shared/extra-utils/server/stats' |
37 | import { ServerStats } from '../../../../shared/models/server/server-stats.model' | 43 | import { ServerStats } from '../../../../shared/models/server/server-stats.model' |
38 | 44 | ||
@@ -40,6 +46,7 @@ const expect = chai.expect | |||
40 | 46 | ||
41 | let servers: ServerInfo[] = [] | 47 | let servers: ServerInfo[] = [] |
42 | let video1Server2UUID: string | 48 | let video1Server2UUID: string |
49 | let video1Server2Id: number | ||
43 | 50 | ||
44 | function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) { | 51 | function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) { |
45 | const parsed = magnetUtil.decode(file.magnetUri) | 52 | const parsed = magnetUtil.decode(file.magnetUri) |
@@ -52,7 +59,19 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe | |||
52 | expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) | 59 | expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) |
53 | } | 60 | } |
54 | 61 | ||
55 | async function flushAndRunServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { | 62 | async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}) { |
63 | const strategies: any[] = [] | ||
64 | |||
65 | if (strategy !== null) { | ||
66 | strategies.push( | ||
67 | immutableAssign({ | ||
68 | min_lifetime: '1 hour', | ||
69 | strategy: strategy, | ||
70 | size: '400KB' | ||
71 | }, additionalParams) | ||
72 | ) | ||
73 | } | ||
74 | |||
56 | const config = { | 75 | const config = { |
57 | transcoding: { | 76 | transcoding: { |
58 | hls: { | 77 | hls: { |
@@ -62,16 +81,11 @@ async function flushAndRunServers (strategy: VideoRedundancyStrategy, additional | |||
62 | redundancy: { | 81 | redundancy: { |
63 | videos: { | 82 | videos: { |
64 | check_interval: '5 seconds', | 83 | check_interval: '5 seconds', |
65 | strategies: [ | 84 | strategies |
66 | immutableAssign({ | ||
67 | min_lifetime: '1 hour', | ||
68 | strategy: strategy, | ||
69 | size: '400KB' | ||
70 | }, additionalParams) | ||
71 | ] | ||
72 | } | 85 | } |
73 | } | 86 | } |
74 | } | 87 | } |
88 | |||
75 | servers = await flushAndRunMultipleServers(3, config) | 89 | servers = await flushAndRunMultipleServers(3, config) |
76 | 90 | ||
77 | // Get the access tokens | 91 | // Get the access tokens |
@@ -80,6 +94,7 @@ async function flushAndRunServers (strategy: VideoRedundancyStrategy, additional | |||
80 | { | 94 | { |
81 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) | 95 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) |
82 | video1Server2UUID = res.body.video.uuid | 96 | video1Server2UUID = res.body.video.uuid |
97 | video1Server2Id = res.body.video.id | ||
83 | 98 | ||
84 | await viewVideo(servers[ 1 ].url, video1Server2UUID) | 99 | await viewVideo(servers[ 1 ].url, video1Server2UUID) |
85 | } | 100 | } |
@@ -216,29 +231,38 @@ async function check1PlaylistRedundancies (videoUUID?: string) { | |||
216 | } | 231 | } |
217 | } | 232 | } |
218 | 233 | ||
219 | async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { | 234 | async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) { |
235 | let totalSize: number = null | ||
236 | let statsLength = 1 | ||
237 | |||
238 | if (strategy !== 'manual') { | ||
239 | totalSize = 409600 | ||
240 | statsLength = 2 | ||
241 | } | ||
242 | |||
220 | const res = await getStats(servers[0].url) | 243 | const res = await getStats(servers[0].url) |
221 | const data: ServerStats = res.body | 244 | const data: ServerStats = res.body |
222 | 245 | ||
223 | expect(data.videosRedundancy).to.have.lengthOf(1) | 246 | expect(data.videosRedundancy).to.have.lengthOf(statsLength) |
224 | const stat = data.videosRedundancy[0] | ||
225 | 247 | ||
248 | const stat = data.videosRedundancy[0] | ||
226 | expect(stat.strategy).to.equal(strategy) | 249 | expect(stat.strategy).to.equal(strategy) |
227 | expect(stat.totalSize).to.equal(409600) | 250 | expect(stat.totalSize).to.equal(totalSize) |
251 | |||
252 | return stat | ||
253 | } | ||
254 | |||
255 | async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategyWithManual) { | ||
256 | const stat = await checkStatsGlobal(strategy) | ||
257 | |||
228 | expect(stat.totalUsed).to.be.at.least(1).and.below(409601) | 258 | expect(stat.totalUsed).to.be.at.least(1).and.below(409601) |
229 | expect(stat.totalVideoFiles).to.equal(4) | 259 | expect(stat.totalVideoFiles).to.equal(4) |
230 | expect(stat.totalVideos).to.equal(1) | 260 | expect(stat.totalVideos).to.equal(1) |
231 | } | 261 | } |
232 | 262 | ||
233 | async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { | 263 | async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategyWithManual) { |
234 | const res = await getStats(servers[0].url) | 264 | const stat = await checkStatsGlobal(strategy) |
235 | const data: ServerStats = res.body | ||
236 | |||
237 | expect(data.videosRedundancy).to.have.lengthOf(1) | ||
238 | 265 | ||
239 | const stat = data.videosRedundancy[0] | ||
240 | expect(stat.strategy).to.equal(strategy) | ||
241 | expect(stat.totalSize).to.equal(409600) | ||
242 | expect(stat.totalUsed).to.equal(0) | 266 | expect(stat.totalUsed).to.equal(0) |
243 | expect(stat.totalVideoFiles).to.equal(0) | 267 | expect(stat.totalVideoFiles).to.equal(0) |
244 | expect(stat.totalVideos).to.equal(0) | 268 | expect(stat.totalVideos).to.equal(0) |
@@ -446,6 +470,74 @@ describe('Test videos redundancy', function () { | |||
446 | }) | 470 | }) |
447 | }) | 471 | }) |
448 | 472 | ||
473 | describe('With manual strategy', function () { | ||
474 | before(function () { | ||
475 | this.timeout(120000) | ||
476 | |||
477 | return flushAndRunServers(null) | ||
478 | }) | ||
479 | |||
480 | it('Should have 1 webseed on the first video', async function () { | ||
481 | await check1WebSeed() | ||
482 | await check0PlaylistRedundancies() | ||
483 | await checkStatsWith1Webseed('manual') | ||
484 | }) | ||
485 | |||
486 | it('Should create a redundancy on first video', async function () { | ||
487 | await addVideoRedundancy({ | ||
488 | url: servers[0].url, | ||
489 | accessToken: servers[0].accessToken, | ||
490 | videoId: video1Server2Id | ||
491 | }) | ||
492 | }) | ||
493 | |||
494 | it('Should have 2 webseeds on the first video', async function () { | ||
495 | this.timeout(80000) | ||
496 | |||
497 | await waitJobs(servers) | ||
498 | await waitUntilLog(servers[0], 'Duplicated ', 5) | ||
499 | await waitJobs(servers) | ||
500 | |||
501 | await check2Webseeds() | ||
502 | await check1PlaylistRedundancies() | ||
503 | await checkStatsWith2Webseed('manual') | ||
504 | }) | ||
505 | |||
506 | it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () { | ||
507 | this.timeout(80000) | ||
508 | |||
509 | const res = await listVideoRedundancies({ | ||
510 | url: servers[0].url, | ||
511 | accessToken: servers[0].accessToken, | ||
512 | target: 'remote-videos' | ||
513 | }) | ||
514 | |||
515 | const videos = res.body.data as VideoRedundancy[] | ||
516 | expect(videos).to.have.lengthOf(1) | ||
517 | |||
518 | const video = videos[0] | ||
519 | for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) { | ||
520 | await removeVideoRedundancy({ | ||
521 | url: servers[0].url, | ||
522 | accessToken: servers[0].accessToken, | ||
523 | redundancyId: r.id | ||
524 | }) | ||
525 | } | ||
526 | |||
527 | await waitJobs(servers) | ||
528 | await wait(5000) | ||
529 | |||
530 | await check1WebSeed() | ||
531 | await check0PlaylistRedundancies() | ||
532 | |||
533 | await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) | ||
534 | }) | ||
535 | |||
536 | after(async function () { | ||
537 | await cleanupTests(servers) | ||
538 | }) | ||
539 | }) | ||
540 | |||
449 | describe('Test expiration', function () { | 541 | describe('Test expiration', function () { |
450 | const strategy = 'recently-added' | 542 | const strategy = 'recently-added' |
451 | 543 | ||
diff --git a/server/typings/models/video/video-file.ts b/server/typings/models/video/video-file.ts index 352fe3d32..139b22b2c 100644 --- a/server/typings/models/video/video-file.ts +++ b/server/typings/models/video/video-file.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { VideoFileModel } from '../../../models/video/video-file' | 1 | import { VideoFileModel } from '../../../models/video/video-file' |
2 | import { PickWith, PickWithOpt } from '../../utils' | 2 | import { PickWith, PickWithOpt } from '../../utils' |
3 | import { MVideo, MVideoUUID } from './video' | 3 | import { MVideo, MVideoUUID } from './video' |
4 | import { MVideoRedundancyFileUrl } from './video-redundancy' | 4 | import { MVideoRedundancy, MVideoRedundancyFileUrl } from './video-redundancy' |
5 | import { MStreamingPlaylistVideo, MStreamingPlaylist } from './video-streaming-playlist' | 5 | import { MStreamingPlaylistVideo, MStreamingPlaylist } from './video-streaming-playlist' |
6 | 6 | ||
7 | type Use<K extends keyof VideoFileModel, M> = PickWith<VideoFileModel, K, M> | 7 | type Use<K extends keyof VideoFileModel, M> = PickWith<VideoFileModel, K, M> |
@@ -22,6 +22,9 @@ export type MVideoFileStreamingPlaylistVideo = MVideoFile & | |||
22 | export type MVideoFileVideoUUID = MVideoFile & | 22 | export type MVideoFileVideoUUID = MVideoFile & |
23 | Use<'Video', MVideoUUID> | 23 | Use<'Video', MVideoUUID> |
24 | 24 | ||
25 | export type MVideoFileRedundanciesAll = MVideoFile & | ||
26 | PickWithOpt<VideoFileModel, 'RedundancyVideos', MVideoRedundancy[]> | ||
27 | |||
25 | export type MVideoFileRedundanciesOpt = MVideoFile & | 28 | export type MVideoFileRedundanciesOpt = MVideoFile & |
26 | PickWithOpt<VideoFileModel, 'RedundancyVideos', MVideoRedundancyFileUrl[]> | 29 | PickWithOpt<VideoFileModel, 'RedundancyVideos', MVideoRedundancyFileUrl[]> |
27 | 30 | ||
diff --git a/server/typings/models/video/video-streaming-playlist.ts b/server/typings/models/video/video-streaming-playlist.ts index 436c0c072..6fd489945 100644 --- a/server/typings/models/video/video-streaming-playlist.ts +++ b/server/typings/models/video/video-streaming-playlist.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { VideoStreamingPlaylistModel } from '../../../models/video/video-streaming-playlist' | 1 | import { VideoStreamingPlaylistModel } from '../../../models/video/video-streaming-playlist' |
2 | import { PickWith, PickWithOpt } from '../../utils' | 2 | import { PickWith, PickWithOpt } from '../../utils' |
3 | import { MVideoRedundancyFileUrl } from './video-redundancy' | 3 | import { MVideoRedundancyFileUrl, MVideoRedundancy } from './video-redundancy' |
4 | import { MVideo } from './video' | 4 | import { MVideo } from './video' |
5 | import { MVideoFile } from './video-file' | 5 | import { MVideoFile } from './video-file' |
6 | 6 | ||
@@ -20,6 +20,10 @@ export type MStreamingPlaylistFilesVideo = MStreamingPlaylist & | |||
20 | Use<'VideoFiles', MVideoFile[]> & | 20 | Use<'VideoFiles', MVideoFile[]> & |
21 | Use<'Video', MVideo> | 21 | Use<'Video', MVideo> |
22 | 22 | ||
23 | export type MStreamingPlaylistRedundanciesAll = MStreamingPlaylist & | ||
24 | Use<'VideoFiles', MVideoFile[]> & | ||
25 | Use<'RedundancyVideos', MVideoRedundancy[]> | ||
26 | |||
23 | export type MStreamingPlaylistRedundancies = MStreamingPlaylist & | 27 | export type MStreamingPlaylistRedundancies = MStreamingPlaylist & |
24 | Use<'VideoFiles', MVideoFile[]> & | 28 | Use<'VideoFiles', MVideoFile[]> & |
25 | Use<'RedundancyVideos', MVideoRedundancyFileUrl[]> | 29 | Use<'RedundancyVideos', MVideoRedundancyFileUrl[]> |
diff --git a/server/typings/models/video/video.ts b/server/typings/models/video/video.ts index 7f69a91de..bcc5e5028 100644 --- a/server/typings/models/video/video.ts +++ b/server/typings/models/video/video.ts | |||
@@ -10,8 +10,13 @@ import { | |||
10 | } from './video-channels' | 10 | } from './video-channels' |
11 | import { MTag } from './tag' | 11 | import { MTag } from './tag' |
12 | import { MVideoCaptionLanguage } from './video-caption' | 12 | import { MVideoCaptionLanguage } from './video-caption' |
13 | import { MStreamingPlaylistFiles, MStreamingPlaylistRedundancies, MStreamingPlaylistRedundanciesOpt } from './video-streaming-playlist' | 13 | import { |
14 | import { MVideoFile, MVideoFileRedundanciesOpt } from './video-file' | 14 | MStreamingPlaylistFiles, |
15 | MStreamingPlaylistRedundancies, | ||
16 | MStreamingPlaylistRedundanciesAll, | ||
17 | MStreamingPlaylistRedundanciesOpt | ||
18 | } from './video-streaming-playlist' | ||
19 | import { MVideoFile, MVideoFileRedundanciesAll, MVideoFileRedundanciesOpt } from './video-file' | ||
15 | import { MThumbnail } from './thumbnail' | 20 | import { MThumbnail } from './thumbnail' |
16 | import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist' | 21 | import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist' |
17 | import { MScheduleVideoUpdate } from './schedule-video-update' | 22 | import { MScheduleVideoUpdate } from './schedule-video-update' |
@@ -158,6 +163,10 @@ export type MVideoForUser = MVideo & | |||
158 | Use<'VideoBlacklist', MVideoBlacklistLight> & | 163 | Use<'VideoBlacklist', MVideoBlacklistLight> & |
159 | Use<'Thumbnails', MThumbnail[]> | 164 | Use<'Thumbnails', MThumbnail[]> |
160 | 165 | ||
166 | export type MVideoForRedundancyAPI = MVideo & | ||
167 | Use<'VideoFiles', MVideoFileRedundanciesAll[]> & | ||
168 | Use<'VideoStreamingPlaylists', MStreamingPlaylistRedundanciesAll[]> | ||
169 | |||
161 | // ############################################################################ | 170 | // ############################################################################ |
162 | 171 | ||
163 | // Format for API or AP object | 172 | // Format for API or AP object |
diff --git a/shared/extra-utils/server/redundancy.ts b/shared/extra-utils/server/redundancy.ts index c39ff2c8b..7b488e23e 100644 --- a/shared/extra-utils/server/redundancy.ts +++ b/shared/extra-utils/server/redundancy.ts | |||
@@ -1,6 +1,7 @@ | |||
1 | import { makePutBodyRequest } from '../requests/requests' | 1 | import { makeDeleteRequest, makeGetRequest, makePostBodyRequest, makePutBodyRequest } from '../requests/requests' |
2 | import { VideoRedundanciesTarget } from '@shared/models' | ||
2 | 3 | ||
3 | async function updateRedundancy (url: string, accessToken: string, host: string, redundancyAllowed: boolean, expectedStatus = 204) { | 4 | function updateRedundancy (url: string, accessToken: string, host: string, redundancyAllowed: boolean, expectedStatus = 204) { |
4 | const path = '/api/v1/server/redundancy/' + host | 5 | const path = '/api/v1/server/redundancy/' + host |
5 | 6 | ||
6 | return makePutBodyRequest({ | 7 | return makePutBodyRequest({ |
@@ -12,6 +13,69 @@ async function updateRedundancy (url: string, accessToken: string, host: string, | |||
12 | }) | 13 | }) |
13 | } | 14 | } |
14 | 15 | ||
16 | function listVideoRedundancies (options: { | ||
17 | url: string | ||
18 | accessToken: string, | ||
19 | target: VideoRedundanciesTarget, | ||
20 | start?: number, | ||
21 | count?: number, | ||
22 | sort?: string, | ||
23 | statusCodeExpected?: number | ||
24 | }) { | ||
25 | const path = '/api/v1/server/redundancy/videos' | ||
26 | |||
27 | const { url, accessToken, target, statusCodeExpected, start, count, sort } = options | ||
28 | |||
29 | return makeGetRequest({ | ||
30 | url, | ||
31 | token: accessToken, | ||
32 | path, | ||
33 | query: { | ||
34 | start: start ?? 0, | ||
35 | count: count ?? 5, | ||
36 | sort: sort ?? 'name', | ||
37 | target | ||
38 | }, | ||
39 | statusCodeExpected: statusCodeExpected || 200 | ||
40 | }) | ||
41 | } | ||
42 | |||
43 | function addVideoRedundancy (options: { | ||
44 | url: string, | ||
45 | accessToken: string, | ||
46 | videoId: number | ||
47 | }) { | ||
48 | const path = '/api/v1/server/redundancy/videos' | ||
49 | const { url, accessToken, videoId } = options | ||
50 | |||
51 | return makePostBodyRequest({ | ||
52 | url, | ||
53 | token: accessToken, | ||
54 | path, | ||
55 | fields: { videoId }, | ||
56 | statusCodeExpected: 204 | ||
57 | }) | ||
58 | } | ||
59 | |||
60 | function removeVideoRedundancy (options: { | ||
61 | url: string, | ||
62 | accessToken: string, | ||
63 | redundancyId: number | ||
64 | }) { | ||
65 | const { url, accessToken, redundancyId } = options | ||
66 | const path = '/api/v1/server/redundancy/videos/' + redundancyId | ||
67 | |||
68 | return makeDeleteRequest({ | ||
69 | url, | ||
70 | token: accessToken, | ||
71 | path, | ||
72 | statusCodeExpected: 204 | ||
73 | }) | ||
74 | } | ||
75 | |||
15 | export { | 76 | export { |
16 | updateRedundancy | 77 | updateRedundancy, |
78 | listVideoRedundancies, | ||
79 | addVideoRedundancy, | ||
80 | removeVideoRedundancy | ||
17 | } | 81 | } |
diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts index 7a77a03ad..aa13273ae 100644 --- a/shared/extra-utils/videos/videos.ts +++ b/shared/extra-utils/videos/videos.ts | |||
@@ -607,15 +607,28 @@ async function videoUUIDToId (url: string, id: number | string) { | |||
607 | return res.body.id | 607 | return res.body.id |
608 | } | 608 | } |
609 | 609 | ||
610 | async function uploadVideoAndGetId (options: { server: ServerInfo, videoName: string, nsfw?: boolean, token?: string }) { | 610 | async function uploadVideoAndGetId (options: { |
611 | server: ServerInfo, | ||
612 | videoName: string, | ||
613 | nsfw?: boolean, | ||
614 | privacy?: VideoPrivacy, | ||
615 | token?: string | ||
616 | }) { | ||
611 | const videoAttrs: any = { name: options.videoName } | 617 | const videoAttrs: any = { name: options.videoName } |
612 | if (options.nsfw) videoAttrs.nsfw = options.nsfw | 618 | if (options.nsfw) videoAttrs.nsfw = options.nsfw |
619 | if (options.privacy) videoAttrs.privacy = options.privacy | ||
613 | 620 | ||
614 | const res = await uploadVideo(options.server.url, options.token || options.server.accessToken, videoAttrs) | 621 | const res = await uploadVideo(options.server.url, options.token || options.server.accessToken, videoAttrs) |
615 | 622 | ||
616 | return { id: res.body.video.id, uuid: res.body.video.uuid } | 623 | return { id: res.body.video.id, uuid: res.body.video.uuid } |
617 | } | 624 | } |
618 | 625 | ||
626 | async function getLocalIdByUUID (url: string, uuid: string) { | ||
627 | const res = await getVideo(url, uuid) | ||
628 | |||
629 | return res.body.id | ||
630 | } | ||
631 | |||
619 | // --------------------------------------------------------------------------- | 632 | // --------------------------------------------------------------------------- |
620 | 633 | ||
621 | export { | 634 | export { |
@@ -645,5 +658,6 @@ export { | |||
645 | completeVideoCheck, | 658 | completeVideoCheck, |
646 | checkVideoFilesWereRemoved, | 659 | checkVideoFilesWereRemoved, |
647 | getPlaylistVideos, | 660 | getPlaylistVideos, |
648 | uploadVideoAndGetId | 661 | uploadVideoAndGetId, |
662 | getLocalIdByUUID | ||
649 | } | 663 | } |
diff --git a/shared/models/redundancy/index.ts b/shared/models/redundancy/index.ts index 61bf0fca7..649cc489f 100644 --- a/shared/models/redundancy/index.ts +++ b/shared/models/redundancy/index.ts | |||
@@ -1 +1,3 @@ | |||
1 | export * from './videos-redundancy.model' | 1 | export * from './videos-redundancy-strategy.model' |
2 | export * from './video-redundancies-filters.model' | ||
3 | export * from './video-redundancy.model' | ||
diff --git a/shared/models/redundancy/video-redundancies-filters.model.ts b/shared/models/redundancy/video-redundancies-filters.model.ts new file mode 100644 index 000000000..05ba7dfd3 --- /dev/null +++ b/shared/models/redundancy/video-redundancies-filters.model.ts | |||
@@ -0,0 +1 @@ | |||
export type VideoRedundanciesTarget = 'my-videos' | 'remote-videos' | |||
diff --git a/shared/models/redundancy/video-redundancy.model.ts b/shared/models/redundancy/video-redundancy.model.ts new file mode 100644 index 000000000..014f69634 --- /dev/null +++ b/shared/models/redundancy/video-redundancy.model.ts | |||
@@ -0,0 +1,33 @@ | |||
1 | export interface VideoRedundancy { | ||
2 | id: number | ||
3 | name: string | ||
4 | url: string | ||
5 | uuid: string | ||
6 | |||
7 | redundancies: { | ||
8 | files: FileRedundancyInformation[] | ||
9 | |||
10 | streamingPlaylists: StreamingPlaylistRedundancyInformation[] | ||
11 | } | ||
12 | } | ||
13 | |||
14 | interface RedundancyInformation { | ||
15 | id: number | ||
16 | fileUrl: string | ||
17 | strategy: string | ||
18 | |||
19 | createdAt: Date | string | ||
20 | updatedAt: Date | string | ||
21 | |||
22 | expiresOn: Date | string | ||
23 | |||
24 | size: number | ||
25 | } | ||
26 | |||
27 | export interface FileRedundancyInformation extends RedundancyInformation { | ||
28 | |||
29 | } | ||
30 | |||
31 | export interface StreamingPlaylistRedundancyInformation extends RedundancyInformation { | ||
32 | |||
33 | } | ||
diff --git a/shared/models/redundancy/videos-redundancy.model.ts b/shared/models/redundancy/videos-redundancy-strategy.model.ts index a8c2743c1..15409abf0 100644 --- a/shared/models/redundancy/videos-redundancy.model.ts +++ b/shared/models/redundancy/videos-redundancy-strategy.model.ts | |||
@@ -1,4 +1,5 @@ | |||
1 | export type VideoRedundancyStrategy = 'most-views' | 'trending' | 'recently-added' | 1 | export type VideoRedundancyStrategy = 'most-views' | 'trending' | 'recently-added' |
2 | export type VideoRedundancyStrategyWithManual = VideoRedundancyStrategy | 'manual' | ||
2 | 3 | ||
3 | export type MostViewsRedundancyStrategy = { | 4 | export type MostViewsRedundancyStrategy = { |
4 | strategy: 'most-views' | 5 | strategy: 'most-views' |
@@ -19,4 +20,4 @@ export type RecentlyAddedStrategy = { | |||
19 | minLifetime: number | 20 | minLifetime: number |
20 | } | 21 | } |
21 | 22 | ||
22 | export type VideosRedundancy = MostViewsRedundancyStrategy | TrendingRedundancyStrategy | RecentlyAddedStrategy | 23 | export type VideosRedundancyStrategy = MostViewsRedundancyStrategy | TrendingRedundancyStrategy | RecentlyAddedStrategy |
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index b82a633b2..19fd4c659 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts | |||
@@ -9,7 +9,8 @@ export type JobType = 'activitypub-http-unicast' | | |||
9 | 'email' | | 9 | 'email' | |
10 | 'video-import' | | 10 | 'video-import' | |
11 | 'videos-views' | | 11 | 'videos-views' | |
12 | 'activitypub-refresher' | 12 | 'activitypub-refresher' | |
13 | 'video-redundancy' | ||
13 | 14 | ||
14 | export interface Job { | 15 | export interface Job { |
15 | id: number | 16 | id: number |
diff --git a/shared/models/server/server-stats.model.ts b/shared/models/server/server-stats.model.ts index 74f3de5d3..11778e6ed 100644 --- a/shared/models/server/server-stats.model.ts +++ b/shared/models/server/server-stats.model.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { VideoRedundancyStrategy } from '../redundancy' | 1 | import { VideoRedundancyStrategyWithManual } from '../redundancy' |
2 | 2 | ||
3 | export interface ServerStats { | 3 | export interface ServerStats { |
4 | totalUsers: number | 4 | totalUsers: number |
@@ -13,11 +13,13 @@ export interface ServerStats { | |||
13 | totalInstanceFollowers: number | 13 | totalInstanceFollowers: number |
14 | totalInstanceFollowing: number | 14 | totalInstanceFollowing: number |
15 | 15 | ||
16 | videosRedundancy: { | 16 | videosRedundancy: VideosRedundancyStats[] |
17 | strategy: VideoRedundancyStrategy | 17 | } |
18 | totalSize: number | 18 | |
19 | totalUsed: number | 19 | export interface VideosRedundancyStats { |
20 | totalVideoFiles: number | 20 | strategy: VideoRedundancyStrategyWithManual |
21 | totalVideos: number | 21 | totalSize: number |
22 | }[] | 22 | totalUsed: number |
23 | totalVideoFiles: number | ||
24 | totalVideos: number | ||
23 | } | 25 | } |
diff --git a/shared/models/users/user-right.enum.ts b/shared/models/users/user-right.enum.ts index 4a28a229d..2f88a65de 100644 --- a/shared/models/users/user-right.enum.ts +++ b/shared/models/users/user-right.enum.ts | |||
@@ -33,5 +33,7 @@ export enum UserRight { | |||
33 | SEE_ALL_VIDEOS, | 33 | SEE_ALL_VIDEOS, |
34 | CHANGE_VIDEO_OWNERSHIP, | 34 | CHANGE_VIDEO_OWNERSHIP, |
35 | 35 | ||
36 | MANAGE_PLUGINS | 36 | MANAGE_PLUGINS, |
37 | |||
38 | MANAGE_VIDEOS_REDUNDANCIES | ||
37 | } | 39 | } |
diff --git a/shared/models/videos/video.model.ts b/shared/models/videos/video.model.ts index 7576439fe..a69152759 100644 --- a/shared/models/videos/video.model.ts +++ b/shared/models/videos/video.model.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { AccountSummary, VideoChannelSummary, VideoResolution, VideoState } from '../../index' | 1 | import { AccountSummary, VideoChannelSummary, VideoState } from '../../index' |
2 | import { Account } from '../actors' | 2 | import { Account } from '../actors' |
3 | import { VideoChannel } from './channel/video-channel.model' | 3 | import { VideoChannel } from './channel/video-channel.model' |
4 | import { VideoPrivacy } from './video-privacy.enum' | 4 | import { VideoPrivacy } from './video-privacy.enum' |