diff options
author | Florent <florent.git@zeteo.me> | 2022-08-10 09:53:39 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-08-10 09:53:39 +0200 |
commit | 2a491182e483b97afb1b65c908b23cb48d591807 (patch) | |
tree | ec13503216ad72a3ea8f1ce3b659899f8167fb47 /server/tests/api | |
parent | 06ac128958c489efe1008eeca1df683819bd2f18 (diff) | |
download | PeerTube-2a491182e483b97afb1b65c908b23cb48d591807.tar.gz PeerTube-2a491182e483b97afb1b65c908b23cb48d591807.tar.zst PeerTube-2a491182e483b97afb1b65c908b23cb48d591807.zip |
Channel sync (#5135)
* Add external channel URL for channel update / creation (#754)
* Disallow synchronisation if user has no video quota (#754)
* More constraints serverside (#754)
* Disable sync if server configuration does not allow HTTP import (#754)
* Working version synchronizing videos with a job (#754)
TODO: refactoring, too much code duplication
* More logs and try/catch (#754)
* Fix eslint error (#754)
* WIP: support synchronization time change (#754)
* New frontend #754
* WIP: Create sync front (#754)
* Enhance UI, sync creation form (#754)
* Warning message when HTTP upload is disallowed
* More consistent names (#754)
* Binding Front with API (#754)
* Add a /me API (#754)
* Improve list UI (#754)
* Implement creation and deletion routes (#754)
* Lint (#754)
* Lint again (#754)
* WIP: UI for triggering import existing videos (#754)
* Implement jobs for syncing and importing channels
* Don't sync videos before sync creation + avoid concurrency issue (#754)
* Cleanup (#754)
* Cleanup: OpenAPI + API rework (#754)
* Remove dead code (#754)
* Eslint (#754)
* Revert the mess with whitespaces in constants.ts (#754)
* Some fixes after rebase (#754)
* Several fixes after PR remarks (#754)
* Front + API: Rename video-channels-sync to video-channel-syncs (#754)
* Allow enabling channel sync through UI (#754)
* getChannelInfo (#754)
* Minor fixes: openapi + model + sql (#754)
* Simplified API validators (#754)
* Rename MChannelSync to MChannelSyncChannel (#754)
* Add command for VideoChannelSync (#754)
* Use synchronization.enabled config (#754)
* Check parameters test + some fixes (#754)
* Fix conflict mistake (#754)
* Restrict access to video channel sync list API (#754)
* Start adding unit test for synchronization (#754)
* Continue testing (#754)
* Tests finished + convertion of job to scheduler (#754)
* Add lastSyncAt field (#754)
* Fix externalRemoteUrl sort + creation date not well formatted (#754)
* Small fix (#754)
* Factorize addYoutubeDLImport and buildVideo (#754)
* Check duplicates on channel not on users (#754)
* factorize thumbnail generation (#754)
* Fetch error should return status 400 (#754)
* Separate video-channel-import and video-channel-sync-latest (#754)
* Bump DB migration version after rebase (#754)
* Prettier states in UI table (#754)
* Add DefaultScope in VideoChannelSyncModel (#754)
* Fix audit logs (#754)
* Ensure user can upload when importing channel + minor fixes (#754)
* Mark synchronization as failed on exception + typos (#754)
* Change REST API for importing videos into channel (#754)
* Add option for fully synchronize a chnanel (#754)
* Return a whole sync object on creation to avoid tricks in Front (#754)
* Various remarks (#754)
* Single quotes by default (#754)
* Rename synchronization to video_channel_synchronization
* Add check.latest_videos_count and max_per_user options (#754)
* Better channel rendering in list #754
* Allow sorting with channel name and state (#754)
* Add missing tests for channel imports (#754)
* Prefer using a parent job for channel sync
* Styling
* Client styling
Co-authored-by: Chocobozzz <me@florianbigard.com>
Diffstat (limited to 'server/tests/api')
-rw-r--r-- | server/tests/api/check-params/config.ts | 29 | ||||
-rw-r--r-- | server/tests/api/check-params/index.ts | 1 | ||||
-rw-r--r-- | server/tests/api/check-params/upload-quota.ts | 2 | ||||
-rw-r--r-- | server/tests/api/check-params/video-channel-syncs.ts | 318 | ||||
-rw-r--r-- | server/tests/api/check-params/video-channels.ts | 134 | ||||
-rw-r--r-- | server/tests/api/check-params/video-imports.ts | 8 | ||||
-rw-r--r-- | server/tests/api/server/config.ts | 4 | ||||
-rw-r--r-- | server/tests/api/videos/channel-import-videos.ts | 50 | ||||
-rw-r--r-- | server/tests/api/videos/index.ts | 2 | ||||
-rw-r--r-- | server/tests/api/videos/video-channel-syncs.ts | 226 | ||||
-rw-r--r-- | server/tests/api/videos/video-imports.ts | 22 |
11 files changed, 764 insertions, 32 deletions
diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts index 2f9f553ab..d67e51123 100644 --- a/server/tests/api/check-params/config.ts +++ b/server/tests/api/check-params/config.ts | |||
@@ -1,7 +1,8 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | 1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | 2 | ||
3 | import 'mocha' | 3 | import 'mocha' |
4 | import { omit } from 'lodash' | 4 | import { merge, omit } from 'lodash' |
5 | import { CustomConfig, HttpStatusCode } from '@shared/models' | ||
5 | import { | 6 | import { |
6 | cleanupTests, | 7 | cleanupTests, |
7 | createSingleServer, | 8 | createSingleServer, |
@@ -11,7 +12,6 @@ import { | |||
11 | PeerTubeServer, | 12 | PeerTubeServer, |
12 | setAccessTokensToServers | 13 | setAccessTokensToServers |
13 | } from '@shared/server-commands' | 14 | } from '@shared/server-commands' |
14 | import { CustomConfig, HttpStatusCode } from '@shared/models' | ||
15 | 15 | ||
16 | describe('Test config API validators', function () { | 16 | describe('Test config API validators', function () { |
17 | const path = '/api/v1/config/custom' | 17 | const path = '/api/v1/config/custom' |
@@ -162,6 +162,10 @@ describe('Test config API validators', function () { | |||
162 | torrent: { | 162 | torrent: { |
163 | enabled: false | 163 | enabled: false |
164 | } | 164 | } |
165 | }, | ||
166 | videoChannelSynchronization: { | ||
167 | enabled: false, | ||
168 | maxPerUser: 10 | ||
165 | } | 169 | } |
166 | }, | 170 | }, |
167 | trending: { | 171 | trending: { |
@@ -346,7 +350,26 @@ describe('Test config API validators', function () { | |||
346 | }) | 350 | }) |
347 | }) | 351 | }) |
348 | 352 | ||
349 | it('Should success with the correct parameters', async function () { | 353 | it('Should fail with a disabled http upload & enabled sync', async function () { |
354 | const newUpdateParams: CustomConfig = merge({}, updateParams, { | ||
355 | import: { | ||
356 | videos: { | ||
357 | http: { enabled: false } | ||
358 | }, | ||
359 | videoChannelSynchronization: { enabled: true } | ||
360 | } | ||
361 | }) | ||
362 | |||
363 | await makePutBodyRequest({ | ||
364 | url: server.url, | ||
365 | path, | ||
366 | fields: newUpdateParams, | ||
367 | token: server.accessToken, | ||
368 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
369 | }) | ||
370 | }) | ||
371 | |||
372 | it('Should succeed with the correct parameters', async function () { | ||
350 | await makePutBodyRequest({ | 373 | await makePutBodyRequest({ |
351 | url: server.url, | 374 | url: server.url, |
352 | path, | 375 | path, |
diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts index a27bc8509..5f1168b53 100644 --- a/server/tests/api/check-params/index.ts +++ b/server/tests/api/check-params/index.ts | |||
@@ -27,6 +27,7 @@ import './video-channels' | |||
27 | import './video-comments' | 27 | import './video-comments' |
28 | import './video-files' | 28 | import './video-files' |
29 | import './video-imports' | 29 | import './video-imports' |
30 | import './video-channel-syncs' | ||
30 | import './video-playlists' | 31 | import './video-playlists' |
31 | import './video-source' | 32 | import './video-source' |
32 | import './video-studio' | 33 | import './video-studio' |
diff --git a/server/tests/api/check-params/upload-quota.ts b/server/tests/api/check-params/upload-quota.ts index deb4a7aa3..f64eafc18 100644 --- a/server/tests/api/check-params/upload-quota.ts +++ b/server/tests/api/check-params/upload-quota.ts | |||
@@ -70,7 +70,7 @@ describe('Test upload quota', function () { | |||
70 | }) | 70 | }) |
71 | 71 | ||
72 | it('Should fail to import with HTTP/Torrent/magnet', async function () { | 72 | it('Should fail to import with HTTP/Torrent/magnet', async function () { |
73 | this.timeout(120000) | 73 | this.timeout(120_000) |
74 | 74 | ||
75 | const baseAttributes = { | 75 | const baseAttributes = { |
76 | channelId: server.store.channel.id, | 76 | channelId: server.store.channel.id, |
diff --git a/server/tests/api/check-params/video-channel-syncs.ts b/server/tests/api/check-params/video-channel-syncs.ts new file mode 100644 index 000000000..bcd8984df --- /dev/null +++ b/server/tests/api/check-params/video-channel-syncs.ts | |||
@@ -0,0 +1,318 @@ | |||
1 | import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, FIXTURE_URLS } from '@server/tests/shared' | ||
2 | import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models' | ||
3 | import { | ||
4 | ChannelSyncsCommand, | ||
5 | createSingleServer, | ||
6 | makePostBodyRequest, | ||
7 | PeerTubeServer, | ||
8 | setAccessTokensToServers, | ||
9 | setDefaultVideoChannel | ||
10 | } from '@shared/server-commands' | ||
11 | |||
12 | describe('Test video channel sync API validator', () => { | ||
13 | const path = '/api/v1/video-channel-syncs' | ||
14 | let server: PeerTubeServer | ||
15 | let command: ChannelSyncsCommand | ||
16 | let rootChannelId: number | ||
17 | let rootChannelSyncId: number | ||
18 | const userInfo = { | ||
19 | accessToken: '', | ||
20 | username: 'user1', | ||
21 | id: -1, | ||
22 | channelId: -1, | ||
23 | syncId: -1 | ||
24 | } | ||
25 | |||
26 | async function withChannelSyncDisabled<T> (callback: () => Promise<T>): Promise<void> { | ||
27 | try { | ||
28 | await server.config.disableChannelSync() | ||
29 | await callback() | ||
30 | } finally { | ||
31 | await server.config.enableChannelSync() | ||
32 | } | ||
33 | } | ||
34 | |||
35 | async function withMaxSyncsPerUser<T> (maxSync: number, callback: () => Promise<T>): Promise<void> { | ||
36 | const origConfig = await server.config.getCustomConfig() | ||
37 | |||
38 | await server.config.updateExistingSubConfig({ | ||
39 | newConfig: { | ||
40 | import: { | ||
41 | videoChannelSynchronization: { | ||
42 | maxPerUser: maxSync | ||
43 | } | ||
44 | } | ||
45 | } | ||
46 | }) | ||
47 | |||
48 | try { | ||
49 | await callback() | ||
50 | } finally { | ||
51 | await server.config.updateCustomConfig({ newCustomConfig: origConfig }) | ||
52 | } | ||
53 | } | ||
54 | |||
55 | before(async function () { | ||
56 | this.timeout(30_000) | ||
57 | |||
58 | server = await createSingleServer(1) | ||
59 | |||
60 | await setAccessTokensToServers([ server ]) | ||
61 | await setDefaultVideoChannel([ server ]) | ||
62 | |||
63 | command = server.channelSyncs | ||
64 | |||
65 | rootChannelId = server.store.channel.id | ||
66 | |||
67 | { | ||
68 | userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username) | ||
69 | |||
70 | const { videoChannels, id: userId } = await server.users.getMyInfo({ token: userInfo.accessToken }) | ||
71 | userInfo.id = userId | ||
72 | userInfo.channelId = videoChannels[0].id | ||
73 | } | ||
74 | |||
75 | await server.config.enableChannelSync() | ||
76 | }) | ||
77 | |||
78 | describe('When creating a sync', function () { | ||
79 | let baseCorrectParams: VideoChannelSyncCreate | ||
80 | |||
81 | before(function () { | ||
82 | baseCorrectParams = { | ||
83 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
84 | videoChannelId: rootChannelId | ||
85 | } | ||
86 | }) | ||
87 | |||
88 | it('Should fail when sync is disabled', async function () { | ||
89 | await withChannelSyncDisabled(async () => { | ||
90 | await command.create({ | ||
91 | token: server.accessToken, | ||
92 | attributes: baseCorrectParams, | ||
93 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
94 | }) | ||
95 | }) | ||
96 | }) | ||
97 | |||
98 | it('Should fail with nothing', async function () { | ||
99 | const fields = {} | ||
100 | await makePostBodyRequest({ | ||
101 | url: server.url, | ||
102 | path, | ||
103 | token: server.accessToken, | ||
104 | fields, | ||
105 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
106 | }) | ||
107 | }) | ||
108 | |||
109 | it('Should fail with no authentication', async function () { | ||
110 | await command.create({ | ||
111 | token: null, | ||
112 | attributes: baseCorrectParams, | ||
113 | expectedStatus: HttpStatusCode.UNAUTHORIZED_401 | ||
114 | }) | ||
115 | }) | ||
116 | |||
117 | it('Should fail without a target url', async function () { | ||
118 | const attributes: VideoChannelSyncCreate = { | ||
119 | ...baseCorrectParams, | ||
120 | externalChannelUrl: null | ||
121 | } | ||
122 | await command.create({ | ||
123 | token: server.accessToken, | ||
124 | attributes, | ||
125 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
126 | }) | ||
127 | }) | ||
128 | |||
129 | it('Should fail without a channelId', async function () { | ||
130 | const attributes: VideoChannelSyncCreate = { | ||
131 | ...baseCorrectParams, | ||
132 | videoChannelId: null | ||
133 | } | ||
134 | await command.create({ | ||
135 | token: server.accessToken, | ||
136 | attributes, | ||
137 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
138 | }) | ||
139 | }) | ||
140 | |||
141 | it('Should fail with a channelId refering nothing', async function () { | ||
142 | const attributes: VideoChannelSyncCreate = { | ||
143 | ...baseCorrectParams, | ||
144 | videoChannelId: 42 | ||
145 | } | ||
146 | await command.create({ | ||
147 | token: server.accessToken, | ||
148 | attributes, | ||
149 | expectedStatus: HttpStatusCode.NOT_FOUND_404 | ||
150 | }) | ||
151 | }) | ||
152 | |||
153 | it('Should fail to create a sync when the user does not own the channel', async function () { | ||
154 | await command.create({ | ||
155 | token: userInfo.accessToken, | ||
156 | attributes: baseCorrectParams, | ||
157 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
158 | }) | ||
159 | }) | ||
160 | |||
161 | it('Should succeed to create a sync with root and for another user\'s channel', async function () { | ||
162 | const { videoChannelSync } = await command.create({ | ||
163 | token: server.accessToken, | ||
164 | attributes: { | ||
165 | ...baseCorrectParams, | ||
166 | videoChannelId: userInfo.channelId | ||
167 | }, | ||
168 | expectedStatus: HttpStatusCode.OK_200 | ||
169 | }) | ||
170 | userInfo.syncId = videoChannelSync.id | ||
171 | }) | ||
172 | |||
173 | it('Should succeed with the correct parameters', async function () { | ||
174 | const { videoChannelSync } = await command.create({ | ||
175 | token: server.accessToken, | ||
176 | attributes: baseCorrectParams, | ||
177 | expectedStatus: HttpStatusCode.OK_200 | ||
178 | }) | ||
179 | rootChannelSyncId = videoChannelSync.id | ||
180 | }) | ||
181 | |||
182 | it('Should fail when the user exceeds allowed number of synchronizations', async function () { | ||
183 | await withMaxSyncsPerUser(1, async () => { | ||
184 | await command.create({ | ||
185 | token: server.accessToken, | ||
186 | attributes: { | ||
187 | ...baseCorrectParams, | ||
188 | videoChannelId: userInfo.channelId | ||
189 | }, | ||
190 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
191 | }) | ||
192 | }) | ||
193 | }) | ||
194 | }) | ||
195 | |||
196 | describe('When listing my channel syncs', function () { | ||
197 | const myPath = '/api/v1/accounts/root/video-channel-syncs' | ||
198 | |||
199 | it('Should fail with a bad start pagination', async function () { | ||
200 | await checkBadStartPagination(server.url, myPath, server.accessToken) | ||
201 | }) | ||
202 | |||
203 | it('Should fail with a bad count pagination', async function () { | ||
204 | await checkBadCountPagination(server.url, myPath, server.accessToken) | ||
205 | }) | ||
206 | |||
207 | it('Should fail with an incorrect sort', async function () { | ||
208 | await checkBadSortPagination(server.url, myPath, server.accessToken) | ||
209 | }) | ||
210 | |||
211 | it('Should succeed with the correct parameters', async function () { | ||
212 | await command.listByAccount({ | ||
213 | accountName: 'root', | ||
214 | token: server.accessToken, | ||
215 | expectedStatus: HttpStatusCode.OK_200 | ||
216 | }) | ||
217 | }) | ||
218 | |||
219 | it('Should fail with no authentication', async function () { | ||
220 | await command.listByAccount({ | ||
221 | accountName: 'root', | ||
222 | token: null, | ||
223 | expectedStatus: HttpStatusCode.UNAUTHORIZED_401 | ||
224 | }) | ||
225 | }) | ||
226 | |||
227 | it('Should fail when a simple user lists another user\'s synchronizations', async function () { | ||
228 | await command.listByAccount({ | ||
229 | accountName: 'root', | ||
230 | token: userInfo.accessToken, | ||
231 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
232 | }) | ||
233 | }) | ||
234 | |||
235 | it('Should succeed when root lists another user\'s synchronizations', async function () { | ||
236 | await command.listByAccount({ | ||
237 | accountName: userInfo.username, | ||
238 | token: server.accessToken, | ||
239 | expectedStatus: HttpStatusCode.OK_200 | ||
240 | }) | ||
241 | }) | ||
242 | |||
243 | it('Should succeed even with synchronization disabled', async function () { | ||
244 | await withChannelSyncDisabled(async function () { | ||
245 | await command.listByAccount({ | ||
246 | accountName: 'root', | ||
247 | token: server.accessToken, | ||
248 | expectedStatus: HttpStatusCode.OK_200 | ||
249 | }) | ||
250 | }) | ||
251 | }) | ||
252 | }) | ||
253 | |||
254 | describe('When triggering deletion', function () { | ||
255 | it('should fail with no authentication', async function () { | ||
256 | await command.delete({ | ||
257 | channelSyncId: userInfo.syncId, | ||
258 | token: null, | ||
259 | expectedStatus: HttpStatusCode.UNAUTHORIZED_401 | ||
260 | }) | ||
261 | }) | ||
262 | |||
263 | it('Should fail when channelSyncId does not refer to any sync', async function () { | ||
264 | await command.delete({ | ||
265 | channelSyncId: 42, | ||
266 | token: server.accessToken, | ||
267 | expectedStatus: HttpStatusCode.NOT_FOUND_404 | ||
268 | }) | ||
269 | }) | ||
270 | |||
271 | it('Should fail when sync is not owned by the user', async function () { | ||
272 | await command.delete({ | ||
273 | channelSyncId: rootChannelSyncId, | ||
274 | token: userInfo.accessToken, | ||
275 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
276 | }) | ||
277 | }) | ||
278 | |||
279 | it('Should succeed when root delete a sync they do not own', async function () { | ||
280 | await command.delete({ | ||
281 | channelSyncId: userInfo.syncId, | ||
282 | token: server.accessToken, | ||
283 | expectedStatus: HttpStatusCode.NO_CONTENT_204 | ||
284 | }) | ||
285 | }) | ||
286 | |||
287 | it('should succeed when user delete a sync they own', async function () { | ||
288 | const { videoChannelSync } = await command.create({ | ||
289 | attributes: { | ||
290 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
291 | videoChannelId: userInfo.channelId | ||
292 | }, | ||
293 | token: server.accessToken, | ||
294 | expectedStatus: HttpStatusCode.OK_200 | ||
295 | }) | ||
296 | |||
297 | await command.delete({ | ||
298 | channelSyncId: videoChannelSync.id, | ||
299 | token: server.accessToken, | ||
300 | expectedStatus: HttpStatusCode.NO_CONTENT_204 | ||
301 | }) | ||
302 | }) | ||
303 | |||
304 | it('Should succeed even when synchronization is disabled', async function () { | ||
305 | await withChannelSyncDisabled(async function () { | ||
306 | await command.delete({ | ||
307 | channelSyncId: rootChannelSyncId, | ||
308 | token: server.accessToken, | ||
309 | expectedStatus: HttpStatusCode.NO_CONTENT_204 | ||
310 | }) | ||
311 | }) | ||
312 | }) | ||
313 | }) | ||
314 | |||
315 | after(async function () { | ||
316 | await server?.kill() | ||
317 | }) | ||
318 | }) | ||
diff --git a/server/tests/api/check-params/video-channels.ts b/server/tests/api/check-params/video-channels.ts index 5c2650fac..337ea1dd4 100644 --- a/server/tests/api/check-params/video-channels.ts +++ b/server/tests/api/check-params/video-channels.ts | |||
@@ -3,8 +3,8 @@ | |||
3 | import 'mocha' | 3 | import 'mocha' |
4 | import * as chai from 'chai' | 4 | import * as chai from 'chai' |
5 | import { omit } from 'lodash' | 5 | import { omit } from 'lodash' |
6 | import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' | 6 | import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, FIXTURE_URLS } from '@server/tests/shared' |
7 | import { buildAbsoluteFixturePath } from '@shared/core-utils' | 7 | import { areHttpImportTestsDisabled, buildAbsoluteFixturePath } from '@shared/core-utils' |
8 | import { HttpStatusCode, VideoChannelUpdate } from '@shared/models' | 8 | import { HttpStatusCode, VideoChannelUpdate } from '@shared/models' |
9 | import { | 9 | import { |
10 | ChannelsCommand, | 10 | ChannelsCommand, |
@@ -23,7 +23,13 @@ const expect = chai.expect | |||
23 | describe('Test video channels API validator', function () { | 23 | describe('Test video channels API validator', function () { |
24 | const videoChannelPath = '/api/v1/video-channels' | 24 | const videoChannelPath = '/api/v1/video-channels' |
25 | let server: PeerTubeServer | 25 | let server: PeerTubeServer |
26 | let accessTokenUser: string | 26 | const userInfo = { |
27 | accessToken: '', | ||
28 | channelName: 'fake_channel', | ||
29 | id: -1, | ||
30 | videoQuota: -1, | ||
31 | videoQuotaDaily: -1 | ||
32 | } | ||
27 | let command: ChannelsCommand | 33 | let command: ChannelsCommand |
28 | 34 | ||
29 | // --------------------------------------------------------------- | 35 | // --------------------------------------------------------------- |
@@ -35,14 +41,15 @@ describe('Test video channels API validator', function () { | |||
35 | 41 | ||
36 | await setAccessTokensToServers([ server ]) | 42 | await setAccessTokensToServers([ server ]) |
37 | 43 | ||
38 | const user = { | 44 | const userCreds = { |
39 | username: 'fake', | 45 | username: 'fake', |
40 | password: 'fake_password' | 46 | password: 'fake_password' |
41 | } | 47 | } |
42 | 48 | ||
43 | { | 49 | { |
44 | await server.users.create({ username: user.username, password: user.password }) | 50 | const user = await server.users.create({ username: userCreds.username, password: userCreds.password }) |
45 | accessTokenUser = await server.login.getAccessToken(user) | 51 | userInfo.id = user.id |
52 | userInfo.accessToken = await server.login.getAccessToken(userCreds) | ||
46 | } | 53 | } |
47 | 54 | ||
48 | command = server.channels | 55 | command = server.channels |
@@ -191,7 +198,7 @@ describe('Test video channels API validator', function () { | |||
191 | await makePutBodyRequest({ | 198 | await makePutBodyRequest({ |
192 | url: server.url, | 199 | url: server.url, |
193 | path, | 200 | path, |
194 | token: accessTokenUser, | 201 | token: userInfo.accessToken, |
195 | fields: baseCorrectParams, | 202 | fields: baseCorrectParams, |
196 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | 203 | expectedStatus: HttpStatusCode.FORBIDDEN_403 |
197 | }) | 204 | }) |
@@ -339,7 +346,7 @@ describe('Test video channels API validator', function () { | |||
339 | }) | 346 | }) |
340 | 347 | ||
341 | it('Should fail with a another user', async function () { | 348 | it('Should fail with a another user', async function () { |
342 | await makeGetRequest({ url: server.url, path, token: accessTokenUser, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | 349 | await makeGetRequest({ url: server.url, path, token: userInfo.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) |
343 | }) | 350 | }) |
344 | 351 | ||
345 | it('Should succeed with the correct params', async function () { | 352 | it('Should succeed with the correct params', async function () { |
@@ -347,13 +354,122 @@ describe('Test video channels API validator', function () { | |||
347 | }) | 354 | }) |
348 | }) | 355 | }) |
349 | 356 | ||
357 | describe('When triggering full synchronization', function () { | ||
358 | |||
359 | it('Should fail when HTTP upload is disabled', async function () { | ||
360 | await server.config.disableImports() | ||
361 | |||
362 | await command.importVideos({ | ||
363 | channelName: 'super_channel', | ||
364 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
365 | token: server.accessToken, | ||
366 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
367 | }) | ||
368 | |||
369 | await server.config.enableImports() | ||
370 | }) | ||
371 | |||
372 | it('Should fail when externalChannelUrl is not provided', async function () { | ||
373 | await command.importVideos({ | ||
374 | channelName: 'super_channel', | ||
375 | externalChannelUrl: null, | ||
376 | token: server.accessToken, | ||
377 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
378 | }) | ||
379 | }) | ||
380 | |||
381 | it('Should fail when externalChannelUrl is malformed', async function () { | ||
382 | await command.importVideos({ | ||
383 | channelName: 'super_channel', | ||
384 | externalChannelUrl: 'not-a-url', | ||
385 | token: server.accessToken, | ||
386 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
387 | }) | ||
388 | }) | ||
389 | |||
390 | it('Should fail with no authentication', async function () { | ||
391 | await command.importVideos({ | ||
392 | channelName: 'super_channel', | ||
393 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
394 | token: null, | ||
395 | expectedStatus: HttpStatusCode.UNAUTHORIZED_401 | ||
396 | }) | ||
397 | }) | ||
398 | |||
399 | it('Should fail when sync is not owned by the user', async function () { | ||
400 | await command.importVideos({ | ||
401 | channelName: 'super_channel', | ||
402 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
403 | token: userInfo.accessToken, | ||
404 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
405 | }) | ||
406 | }) | ||
407 | |||
408 | it('Should fail when the user has no quota', async function () { | ||
409 | await server.users.update({ | ||
410 | userId: userInfo.id, | ||
411 | videoQuota: 0 | ||
412 | }) | ||
413 | |||
414 | await command.importVideos({ | ||
415 | channelName: 'fake_channel', | ||
416 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
417 | token: userInfo.accessToken, | ||
418 | expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413 | ||
419 | }) | ||
420 | |||
421 | await server.users.update({ | ||
422 | userId: userInfo.id, | ||
423 | videoQuota: userInfo.videoQuota | ||
424 | }) | ||
425 | }) | ||
426 | |||
427 | it('Should fail when the user has no daily quota', async function () { | ||
428 | await server.users.update({ | ||
429 | userId: userInfo.id, | ||
430 | videoQuotaDaily: 0 | ||
431 | }) | ||
432 | |||
433 | await command.importVideos({ | ||
434 | channelName: 'fake_channel', | ||
435 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
436 | token: userInfo.accessToken, | ||
437 | expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413 | ||
438 | }) | ||
439 | |||
440 | await server.users.update({ | ||
441 | userId: userInfo.id, | ||
442 | videoQuotaDaily: userInfo.videoQuotaDaily | ||
443 | }) | ||
444 | }) | ||
445 | |||
446 | it('Should succeed when sync is run by its owner', async function () { | ||
447 | if (!areHttpImportTestsDisabled()) return | ||
448 | |||
449 | await command.importVideos({ | ||
450 | channelName: 'fake_channel', | ||
451 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
452 | token: userInfo.accessToken | ||
453 | }) | ||
454 | }) | ||
455 | |||
456 | it('Should succeed when sync is run with root and for another user\'s channel', async function () { | ||
457 | if (!areHttpImportTestsDisabled()) return | ||
458 | |||
459 | await command.importVideos({ | ||
460 | channelName: 'fake_channel', | ||
461 | externalChannelUrl: FIXTURE_URLS.youtubeChannel | ||
462 | }) | ||
463 | }) | ||
464 | }) | ||
465 | |||
350 | describe('When deleting a video channel', function () { | 466 | describe('When deleting a video channel', function () { |
351 | it('Should fail with a non authenticated user', async function () { | 467 | it('Should fail with a non authenticated user', async function () { |
352 | await command.delete({ token: 'coucou', channelName: 'super_channel', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) | 468 | await command.delete({ token: 'coucou', channelName: 'super_channel', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) |
353 | }) | 469 | }) |
354 | 470 | ||
355 | it('Should fail with another authenticated user', async function () { | 471 | it('Should fail with another authenticated user', async function () { |
356 | await command.delete({ token: accessTokenUser, channelName: 'super_channel', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | 472 | await command.delete({ token: userInfo.accessToken, channelName: 'super_channel', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) |
357 | }) | 473 | }) |
358 | 474 | ||
359 | it('Should fail with an unknown video channel id', async function () { | 475 | it('Should fail with an unknown video channel id', async function () { |
diff --git a/server/tests/api/check-params/video-imports.ts b/server/tests/api/check-params/video-imports.ts index 4439810e8..5cdd0d925 100644 --- a/server/tests/api/check-params/video-imports.ts +++ b/server/tests/api/check-params/video-imports.ts | |||
@@ -88,7 +88,13 @@ describe('Test video imports API validator', function () { | |||
88 | 88 | ||
89 | it('Should fail with nothing', async function () { | 89 | it('Should fail with nothing', async function () { |
90 | const fields = {} | 90 | const fields = {} |
91 | await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) | 91 | await makePostBodyRequest({ |
92 | url: server.url, | ||
93 | path, | ||
94 | token: server.accessToken, | ||
95 | fields, | ||
96 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
97 | }) | ||
92 | }) | 98 | }) |
93 | 99 | ||
94 | it('Should fail without a target url', async function () { | 100 | it('Should fail without a target url', async function () { |
diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index efc57b345..fc8711161 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts | |||
@@ -368,6 +368,10 @@ const newCustomConfig: CustomConfig = { | |||
368 | torrent: { | 368 | torrent: { |
369 | enabled: false | 369 | enabled: false |
370 | } | 370 | } |
371 | }, | ||
372 | videoChannelSynchronization: { | ||
373 | enabled: false, | ||
374 | maxPerUser: 10 | ||
371 | } | 375 | } |
372 | }, | 376 | }, |
373 | trending: { | 377 | trending: { |
diff --git a/server/tests/api/videos/channel-import-videos.ts b/server/tests/api/videos/channel-import-videos.ts new file mode 100644 index 000000000..f7540e1ba --- /dev/null +++ b/server/tests/api/videos/channel-import-videos.ts | |||
@@ -0,0 +1,50 @@ | |||
1 | import { expect } from 'chai' | ||
2 | import { FIXTURE_URLS } from '@server/tests/shared' | ||
3 | import { areHttpImportTestsDisabled } from '@shared/core-utils' | ||
4 | import { | ||
5 | createSingleServer, | ||
6 | getServerImportConfig, | ||
7 | PeerTubeServer, | ||
8 | setAccessTokensToServers, | ||
9 | setDefaultVideoChannel, | ||
10 | waitJobs | ||
11 | } from '@shared/server-commands' | ||
12 | |||
13 | describe('Test videos import in a channel', function () { | ||
14 | if (areHttpImportTestsDisabled()) return | ||
15 | |||
16 | function runSuite (mode: 'youtube-dl' | 'yt-dlp') { | ||
17 | |||
18 | describe('Import using ' + mode, function () { | ||
19 | let server: PeerTubeServer | ||
20 | |||
21 | before(async function () { | ||
22 | this.timeout(120_000) | ||
23 | |||
24 | server = await createSingleServer(1, getServerImportConfig(mode)) | ||
25 | |||
26 | await setAccessTokensToServers([ server ]) | ||
27 | await setDefaultVideoChannel([ server ]) | ||
28 | |||
29 | await server.config.enableChannelSync() | ||
30 | }) | ||
31 | |||
32 | it('Should import a whole channel', async function () { | ||
33 | this.timeout(240_000) | ||
34 | |||
35 | await server.channels.importVideos({ channelName: server.store.channel.name, externalChannelUrl: FIXTURE_URLS.youtubeChannel }) | ||
36 | await waitJobs(server) | ||
37 | |||
38 | const videos = await server.videos.listByChannel({ handle: server.store.channel.name }) | ||
39 | expect(videos.total).to.equal(2) | ||
40 | }) | ||
41 | |||
42 | after(async function () { | ||
43 | await server?.kill() | ||
44 | }) | ||
45 | }) | ||
46 | } | ||
47 | |||
48 | runSuite('yt-dlp') | ||
49 | runSuite('youtube-dl') | ||
50 | }) | ||
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts index a0b6b01cf..266155297 100644 --- a/server/tests/api/videos/index.ts +++ b/server/tests/api/videos/index.ts | |||
@@ -4,6 +4,8 @@ import './single-server' | |||
4 | import './video-captions' | 4 | import './video-captions' |
5 | import './video-change-ownership' | 5 | import './video-change-ownership' |
6 | import './video-channels' | 6 | import './video-channels' |
7 | import './channel-import-videos' | ||
8 | import './video-channel-syncs' | ||
7 | import './video-comments' | 9 | import './video-comments' |
8 | import './video-description' | 10 | import './video-description' |
9 | import './video-files' | 11 | import './video-files' |
diff --git a/server/tests/api/videos/video-channel-syncs.ts b/server/tests/api/videos/video-channel-syncs.ts new file mode 100644 index 000000000..229c01f68 --- /dev/null +++ b/server/tests/api/videos/video-channel-syncs.ts | |||
@@ -0,0 +1,226 @@ | |||
1 | import 'mocha' | ||
2 | import { expect } from 'chai' | ||
3 | import { FIXTURE_URLS } from '@server/tests/shared' | ||
4 | import { areHttpImportTestsDisabled } from '@shared/core-utils' | ||
5 | import { HttpStatusCode, VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@shared/models' | ||
6 | import { | ||
7 | ChannelSyncsCommand, | ||
8 | createSingleServer, | ||
9 | getServerImportConfig, | ||
10 | PeerTubeServer, | ||
11 | setAccessTokensToServers, | ||
12 | setDefaultAccountAvatar, | ||
13 | setDefaultChannelAvatar, | ||
14 | setDefaultVideoChannel, | ||
15 | waitJobs | ||
16 | } from '@shared/server-commands' | ||
17 | |||
18 | describe('Test channel synchronizations', function () { | ||
19 | if (areHttpImportTestsDisabled()) return | ||
20 | |||
21 | function runSuite (mode: 'youtube-dl' | 'yt-dlp') { | ||
22 | |||
23 | describe('Sync using ' + mode, function () { | ||
24 | let server: PeerTubeServer | ||
25 | let command: ChannelSyncsCommand | ||
26 | let startTestDate: Date | ||
27 | const userInfo = { | ||
28 | accessToken: '', | ||
29 | username: 'user1', | ||
30 | channelName: 'user1_channel', | ||
31 | channelId: -1, | ||
32 | syncId: -1 | ||
33 | } | ||
34 | |||
35 | async function changeDateForSync (channelSyncId: number, newDate: string) { | ||
36 | await server.sql.updateQuery( | ||
37 | `UPDATE "videoChannelSync" ` + | ||
38 | `SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` + | ||
39 | `WHERE id=${channelSyncId}` | ||
40 | ) | ||
41 | } | ||
42 | |||
43 | before(async function () { | ||
44 | this.timeout(120_000) | ||
45 | |||
46 | startTestDate = new Date() | ||
47 | |||
48 | server = await createSingleServer(1, getServerImportConfig(mode)) | ||
49 | |||
50 | await setAccessTokensToServers([ server ]) | ||
51 | await setDefaultVideoChannel([ server ]) | ||
52 | await setDefaultChannelAvatar([ server ]) | ||
53 | await setDefaultAccountAvatar([ server ]) | ||
54 | |||
55 | await server.config.enableChannelSync() | ||
56 | |||
57 | command = server.channelSyncs | ||
58 | |||
59 | { | ||
60 | userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username) | ||
61 | |||
62 | const { videoChannels } = await server.users.getMyInfo({ token: userInfo.accessToken }) | ||
63 | userInfo.channelId = videoChannels[0].id | ||
64 | } | ||
65 | }) | ||
66 | |||
67 | it('Should fetch the latest channel videos of a remote channel', async function () { | ||
68 | this.timeout(120_000) | ||
69 | |||
70 | { | ||
71 | const { video } = await server.imports.importVideo({ | ||
72 | attributes: { | ||
73 | channelId: server.store.channel.id, | ||
74 | privacy: VideoPrivacy.PUBLIC, | ||
75 | targetUrl: FIXTURE_URLS.youtube | ||
76 | } | ||
77 | }) | ||
78 | |||
79 | expect(video.name).to.equal('small video - youtube') | ||
80 | |||
81 | const { total } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE }) | ||
82 | expect(total).to.equal(1) | ||
83 | } | ||
84 | |||
85 | const { videoChannelSync } = await command.create({ | ||
86 | attributes: { | ||
87 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
88 | videoChannelId: server.store.channel.id | ||
89 | }, | ||
90 | token: server.accessToken, | ||
91 | expectedStatus: HttpStatusCode.OK_200 | ||
92 | }) | ||
93 | |||
94 | // Ensure any missing video not already fetched will be considered as new | ||
95 | await changeDateForSync(videoChannelSync.id, '1970-01-01') | ||
96 | |||
97 | await server.debug.sendCommand({ | ||
98 | body: { | ||
99 | command: 'process-video-channel-sync-latest' | ||
100 | } | ||
101 | }) | ||
102 | |||
103 | { | ||
104 | await waitJobs(server) | ||
105 | |||
106 | const { total, data } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE }) | ||
107 | expect(total).to.equal(2) | ||
108 | expect(data[0].name).to.equal('test') | ||
109 | } | ||
110 | }) | ||
111 | |||
112 | it('Should add another synchronization', async function () { | ||
113 | const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar' | ||
114 | |||
115 | const { videoChannelSync } = await command.create({ | ||
116 | attributes: { | ||
117 | externalChannelUrl, | ||
118 | videoChannelId: server.store.channel.id | ||
119 | }, | ||
120 | token: server.accessToken, | ||
121 | expectedStatus: HttpStatusCode.OK_200 | ||
122 | }) | ||
123 | |||
124 | expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl) | ||
125 | expect(videoChannelSync.channel).to.include({ | ||
126 | id: server.store.channel.id, | ||
127 | name: 'root_channel' | ||
128 | }) | ||
129 | expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN) | ||
130 | expect(new Date(videoChannelSync.createdAt)).to.be.above(startTestDate).and.to.be.at.most(new Date()) | ||
131 | }) | ||
132 | |||
133 | it('Should add a synchronization for another user', async function () { | ||
134 | const { videoChannelSync } = await command.create({ | ||
135 | attributes: { | ||
136 | externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux', | ||
137 | videoChannelId: userInfo.channelId | ||
138 | }, | ||
139 | token: userInfo.accessToken | ||
140 | }) | ||
141 | userInfo.syncId = videoChannelSync.id | ||
142 | }) | ||
143 | |||
144 | it('Should not import a channel if not asked', async function () { | ||
145 | await waitJobs(server) | ||
146 | |||
147 | const { data } = await command.listByAccount({ accountName: userInfo.username }) | ||
148 | |||
149 | expect(data[0].state).to.contain({ | ||
150 | id: VideoChannelSyncState.WAITING_FIRST_RUN, | ||
151 | label: 'Waiting first run' | ||
152 | }) | ||
153 | }) | ||
154 | |||
155 | it('Should only fetch the videos newer than the creation date', async function () { | ||
156 | this.timeout(120_000) | ||
157 | |||
158 | await changeDateForSync(userInfo.syncId, '2019-03-01') | ||
159 | |||
160 | await server.debug.sendCommand({ | ||
161 | body: { | ||
162 | command: 'process-video-channel-sync-latest' | ||
163 | } | ||
164 | }) | ||
165 | |||
166 | await waitJobs(server) | ||
167 | |||
168 | const { data, total } = await server.videos.listByChannel({ | ||
169 | handle: userInfo.channelName, | ||
170 | include: VideoInclude.NOT_PUBLISHED_STATE | ||
171 | }) | ||
172 | |||
173 | expect(total).to.equal(1) | ||
174 | expect(data[0].name).to.equal('test') | ||
175 | }) | ||
176 | |||
177 | it('Should list channel synchronizations', async function () { | ||
178 | // Root | ||
179 | { | ||
180 | const { total, data } = await command.listByAccount({ accountName: 'root' }) | ||
181 | expect(total).to.equal(2) | ||
182 | |||
183 | expect(data[0]).to.deep.contain({ | ||
184 | externalChannelUrl: FIXTURE_URLS.youtubeChannel, | ||
185 | state: { | ||
186 | id: VideoChannelSyncState.SYNCED, | ||
187 | label: 'Synchronized' | ||
188 | } | ||
189 | }) | ||
190 | |||
191 | expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate) | ||
192 | |||
193 | expect(data[0].channel).to.contain({ id: server.store.channel.id }) | ||
194 | expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' }) | ||
195 | } | ||
196 | |||
197 | // User | ||
198 | { | ||
199 | const { total, data } = await command.listByAccount({ accountName: userInfo.username }) | ||
200 | expect(total).to.equal(1) | ||
201 | expect(data[0]).to.deep.contain({ | ||
202 | externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux', | ||
203 | state: { | ||
204 | id: VideoChannelSyncState.SYNCED, | ||
205 | label: 'Synchronized' | ||
206 | } | ||
207 | }) | ||
208 | } | ||
209 | }) | ||
210 | |||
211 | it('Should remove user\'s channel synchronizations', async function () { | ||
212 | await command.delete({ channelSyncId: userInfo.syncId }) | ||
213 | |||
214 | const { total } = await command.listByAccount({ accountName: userInfo.username }) | ||
215 | expect(total).to.equal(0) | ||
216 | }) | ||
217 | |||
218 | after(async function () { | ||
219 | await server?.kill() | ||
220 | }) | ||
221 | }) | ||
222 | } | ||
223 | |||
224 | runSuite('youtube-dl') | ||
225 | runSuite('yt-dlp') | ||
226 | }) | ||
diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts index 603e2d234..a487062a2 100644 --- a/server/tests/api/videos/video-imports.ts +++ b/server/tests/api/videos/video-imports.ts | |||
@@ -12,6 +12,7 @@ import { | |||
12 | createMultipleServers, | 12 | createMultipleServers, |
13 | createSingleServer, | 13 | createSingleServer, |
14 | doubleFollow, | 14 | doubleFollow, |
15 | getServerImportConfig, | ||
15 | PeerTubeServer, | 16 | PeerTubeServer, |
16 | setAccessTokensToServers, | 17 | setAccessTokensToServers, |
17 | setDefaultVideoChannel, | 18 | setDefaultVideoChannel, |
@@ -84,24 +85,9 @@ describe('Test video imports', function () { | |||
84 | let servers: PeerTubeServer[] = [] | 85 | let servers: PeerTubeServer[] = [] |
85 | 86 | ||
86 | before(async function () { | 87 | before(async function () { |
87 | this.timeout(30_000) | 88 | this.timeout(60_000) |
88 | 89 | ||
89 | // Run servers | 90 | servers = await createMultipleServers(2, getServerImportConfig(mode)) |
90 | servers = await createMultipleServers(2, { | ||
91 | import: { | ||
92 | videos: { | ||
93 | http: { | ||
94 | youtube_dl_release: { | ||
95 | url: mode === 'youtube-dl' | ||
96 | ? 'https://yt-dl.org/downloads/latest/youtube-dl' | ||
97 | : 'https://api.github.com/repos/yt-dlp/yt-dlp/releases', | ||
98 | |||
99 | name: mode | ||
100 | } | ||
101 | } | ||
102 | } | ||
103 | } | ||
104 | }) | ||
105 | 91 | ||
106 | await setAccessTokensToServers(servers) | 92 | await setAccessTokensToServers(servers) |
107 | await setDefaultVideoChannel(servers) | 93 | await setDefaultVideoChannel(servers) |