aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2022-09-16 09:37:07 +0200
committerChocobozzz <me@florianbigard.com>2022-09-16 09:37:07 +0200
commit3204f4d17dec79413a94eae8a71b72e0c7ec7ba5 (patch)
treeb6f5a2c3d6e55676972e671fb0ef429a7bdd975b /server
parentc95fbe65530191d7be22341815ecb743c139396f (diff)
downloadPeerTube-3204f4d17dec79413a94eae8a71b72e0c7ec7ba5.tar.gz
PeerTube-3204f4d17dec79413a94eae8a71b72e0c7ec7ba5.tar.zst
PeerTube-3204f4d17dec79413a94eae8a71b72e0c7ec7ba5.zip
Fix sync import of latest videos
Diffstat (limited to 'server')
-rw-r--r--server/helpers/youtube-dl/youtube-dl-info-builder.ts4
-rw-r--r--server/lib/job-queue/handlers/video-channel-import.ts4
-rw-r--r--server/lib/sync-channel.ts5
-rw-r--r--server/lib/video-import.ts2
-rw-r--r--server/tests/api/videos/video-channel-syncs.ts138
5 files changed, 106 insertions, 47 deletions
diff --git a/server/helpers/youtube-dl/youtube-dl-info-builder.ts b/server/helpers/youtube-dl/youtube-dl-info-builder.ts
index 303e4051f..a74904e43 100644
--- a/server/helpers/youtube-dl/youtube-dl-info-builder.ts
+++ b/server/helpers/youtube-dl/youtube-dl-info-builder.ts
@@ -12,7 +12,7 @@ type YoutubeDLInfo = {
12 tags?: string[] 12 tags?: string[]
13 thumbnailUrl?: string 13 thumbnailUrl?: string
14 ext?: string 14 ext?: string
15 originallyPublishedAt?: Date 15 originallyPublishedAtWithoutTime?: Date
16 webpageUrl?: string 16 webpageUrl?: string
17 17
18 urls?: string[] 18 urls?: string[]
@@ -81,7 +81,7 @@ class YoutubeDLInfoBuilder {
81 tags: this.getTags(obj.tags), 81 tags: this.getTags(obj.tags),
82 thumbnailUrl: obj.thumbnail || undefined, 82 thumbnailUrl: obj.thumbnail || undefined,
83 urls: this.buildAvailableUrl(obj), 83 urls: this.buildAvailableUrl(obj),
84 originallyPublishedAt: this.buildOriginallyPublishedAt(obj), 84 originallyPublishedAtWithoutTime: this.buildOriginallyPublishedAt(obj),
85 ext: obj.ext, 85 ext: obj.ext,
86 webpageUrl: obj.webpage_url 86 webpageUrl: obj.webpage_url
87 } 87 }
diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts
index 9aaad659e..600292844 100644
--- a/server/lib/job-queue/handlers/video-channel-import.ts
+++ b/server/lib/job-queue/handlers/video-channel-import.ts
@@ -5,7 +5,7 @@ import { synchronizeChannel } from '@server/lib/sync-channel'
5import { VideoChannelModel } from '@server/models/video/video-channel' 5import { VideoChannelModel } from '@server/models/video/video-channel'
6import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' 6import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
7import { MChannelSync } from '@server/types/models' 7import { MChannelSync } from '@server/types/models'
8import { VideoChannelImportPayload } from '@shared/models' 8import { VideoChannelImportPayload, VideoChannelSyncState } from '@shared/models'
9 9
10export async function processVideoChannelImport (job: Job) { 10export async function processVideoChannelImport (job: Job) {
11 const payload = job.data as VideoChannelImportPayload 11 const payload = job.data as VideoChannelImportPayload
@@ -42,5 +42,7 @@ export async function processVideoChannelImport (job: Job) {
42 }) 42 })
43 } catch (err) { 43 } catch (err) {
44 logger.error(`Failed to import channel ${videoChannel.name}`, { err }) 44 logger.error(`Failed to import channel ${videoChannel.name}`, { err })
45 channelSync.state = VideoChannelSyncState.FAILED
46 await channelSync.save()
45 } 47 }
46} 48}
diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts
index 3a81daac0..6cd1fc26d 100644
--- a/server/lib/sync-channel.ts
+++ b/server/lib/sync-channel.ts
@@ -93,7 +93,10 @@ async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Dat
93 93
94 const videoInfo = await youtubeDL.getInfoForDownload() 94 const videoInfo = await youtubeDL.getInfoForDownload()
95 95
96 if (videoInfo.originallyPublishedAt.getTime() < onlyAfter.getTime()) { 96 const onlyAfterWithoutTime = new Date(onlyAfter)
97 onlyAfterWithoutTime.setHours(0, 0, 0)
98
99 if (videoInfo.originallyPublishedAtWithoutTime.getTime() >= onlyAfterWithoutTime.getTime()) {
97 return true 100 return true
98 } 101 }
99 } 102 }
diff --git a/server/lib/video-import.ts b/server/lib/video-import.ts
index de95116aa..d9da09040 100644
--- a/server/lib/video-import.ts
+++ b/server/lib/video-import.ts
@@ -125,7 +125,7 @@ async function buildVideoFromImport ({ channelId, importData, importDataOverride
125 channelId, 125 channelId,
126 originallyPublishedAt: importDataOverride?.originallyPublishedAt 126 originallyPublishedAt: importDataOverride?.originallyPublishedAt
127 ? new Date(importDataOverride?.originallyPublishedAt) 127 ? new Date(importDataOverride?.originallyPublishedAt)
128 : importData.originallyPublishedAt 128 : importData.originallyPublishedAtWithoutTime
129 } 129 }
130 130
131 videoData = await Hooks.wrapObject( 131 videoData = await Hooks.wrapObject(
diff --git a/server/tests/api/videos/video-channel-syncs.ts b/server/tests/api/videos/video-channel-syncs.ts
index 4e18ddfc7..65b143a4f 100644
--- a/server/tests/api/videos/video-channel-syncs.ts
+++ b/server/tests/api/videos/video-channel-syncs.ts
@@ -3,9 +3,9 @@ import { FIXTURE_URLS } from '@server/tests/shared'
3import { areHttpImportTestsDisabled } from '@shared/core-utils' 3import { areHttpImportTestsDisabled } from '@shared/core-utils'
4import { HttpStatusCode, VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@shared/models' 4import { HttpStatusCode, VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@shared/models'
5import { 5import {
6 ChannelSyncsCommand, 6 createMultipleServers,
7 createSingleServer,
8 getServerImportConfig, 7 getServerImportConfig,
8 killallServers,
9 PeerTubeServer, 9 PeerTubeServer,
10 setAccessTokensToServers, 10 setAccessTokensToServers,
11 setDefaultAccountAvatar, 11 setDefaultAccountAvatar,
@@ -20,8 +20,7 @@ describe('Test channel synchronizations', function () {
20 function runSuite (mode: 'youtube-dl' | 'yt-dlp') { 20 function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
21 21
22 describe('Sync using ' + mode, function () { 22 describe('Sync using ' + mode, function () {
23 let server: PeerTubeServer 23 let servers: PeerTubeServer[]
24 let command: ChannelSyncsCommand
25 24
26 let startTestDate: Date 25 let startTestDate: Date
27 26
@@ -35,7 +34,7 @@ describe('Test channel synchronizations', function () {
35 } 34 }
36 35
37 async function changeDateForSync (channelSyncId: number, newDate: string) { 36 async function changeDateForSync (channelSyncId: number, newDate: string) {
38 await server.sql.updateQuery( 37 await servers[0].sql.updateQuery(
39 `UPDATE "videoChannelSync" ` + 38 `UPDATE "videoChannelSync" ` +
40 `SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` + 39 `SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` +
41 `WHERE id=${channelSyncId}` 40 `WHERE id=${channelSyncId}`
@@ -43,25 +42,23 @@ describe('Test channel synchronizations', function () {
43 } 42 }
44 43
45 before(async function () { 44 before(async function () {
46 this.timeout(120_000) 45 this.timeout(240_000)
47 46
48 startTestDate = new Date() 47 startTestDate = new Date()
49 48
50 server = await createSingleServer(1, getServerImportConfig(mode)) 49 servers = await createMultipleServers(2, getServerImportConfig(mode))
51
52 await setAccessTokensToServers([ server ])
53 await setDefaultVideoChannel([ server ])
54 await setDefaultChannelAvatar([ server ])
55 await setDefaultAccountAvatar([ server ])
56 50
57 await server.config.enableChannelSync() 51 await setAccessTokensToServers(servers)
52 await setDefaultVideoChannel(servers)
53 await setDefaultChannelAvatar(servers)
54 await setDefaultAccountAvatar(servers)
58 55
59 command = server.channelSyncs 56 await servers[0].config.enableChannelSync()
60 57
61 { 58 {
62 userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username) 59 userInfo.accessToken = await servers[0].users.generateUserAndToken(userInfo.username)
63 60
64 const { videoChannels } = await server.users.getMyInfo({ token: userInfo.accessToken }) 61 const { videoChannels } = await servers[0].users.getMyInfo({ token: userInfo.accessToken })
65 userInfo.channelId = videoChannels[0].id 62 userInfo.channelId = videoChannels[0].id
66 } 63 }
67 }) 64 })
@@ -70,9 +67,9 @@ describe('Test channel synchronizations', function () {
70 this.timeout(120_000) 67 this.timeout(120_000)
71 68
72 { 69 {
73 const { video } = await server.imports.importVideo({ 70 const { video } = await servers[0].imports.importVideo({
74 attributes: { 71 attributes: {
75 channelId: server.store.channel.id, 72 channelId: servers[0].store.channel.id,
76 privacy: VideoPrivacy.PUBLIC, 73 privacy: VideoPrivacy.PUBLIC,
77 targetUrl: FIXTURE_URLS.youtube 74 targetUrl: FIXTURE_URLS.youtube
78 } 75 }
@@ -80,16 +77,16 @@ describe('Test channel synchronizations', function () {
80 77
81 expect(video.name).to.equal('small video - youtube') 78 expect(video.name).to.equal('small video - youtube')
82 79
83 const { total } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE }) 80 const { total } = await servers[0].videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
84 expect(total).to.equal(1) 81 expect(total).to.equal(1)
85 } 82 }
86 83
87 const { videoChannelSync } = await command.create({ 84 const { videoChannelSync } = await servers[0].channelSyncs.create({
88 attributes: { 85 attributes: {
89 externalChannelUrl: FIXTURE_URLS.youtubeChannel, 86 externalChannelUrl: FIXTURE_URLS.youtubeChannel,
90 videoChannelId: server.store.channel.id 87 videoChannelId: servers[0].store.channel.id
91 }, 88 },
92 token: server.accessToken, 89 token: servers[0].accessToken,
93 expectedStatus: HttpStatusCode.OK_200 90 expectedStatus: HttpStatusCode.OK_200
94 }) 91 })
95 rootChannelSyncId = videoChannelSync.id 92 rootChannelSyncId = videoChannelSync.id
@@ -97,16 +94,19 @@ describe('Test channel synchronizations', function () {
97 // Ensure any missing video not already fetched will be considered as new 94 // Ensure any missing video not already fetched will be considered as new
98 await changeDateForSync(videoChannelSync.id, '1970-01-01') 95 await changeDateForSync(videoChannelSync.id, '1970-01-01')
99 96
100 await server.debug.sendCommand({ 97 await servers[0].debug.sendCommand({
101 body: { 98 body: {
102 command: 'process-video-channel-sync-latest' 99 command: 'process-video-channel-sync-latest'
103 } 100 }
104 }) 101 })
105 102
106 { 103 {
107 await waitJobs(server) 104 await waitJobs(servers)
108 105
109 const { total, data } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE }) 106 const { total, data } = await servers[0].videos.listByChannel({
107 handle: 'root_channel',
108 include: VideoInclude.NOT_PUBLISHED_STATE
109 })
110 expect(total).to.equal(2) 110 expect(total).to.equal(2)
111 expect(data[0].name).to.equal('test') 111 expect(data[0].name).to.equal('test')
112 } 112 }
@@ -115,18 +115,18 @@ describe('Test channel synchronizations', function () {
115 it('Should add another synchronization', async function () { 115 it('Should add another synchronization', async function () {
116 const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar' 116 const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar'
117 117
118 const { videoChannelSync } = await command.create({ 118 const { videoChannelSync } = await servers[0].channelSyncs.create({
119 attributes: { 119 attributes: {
120 externalChannelUrl, 120 externalChannelUrl,
121 videoChannelId: server.store.channel.id 121 videoChannelId: servers[0].store.channel.id
122 }, 122 },
123 token: server.accessToken, 123 token: servers[0].accessToken,
124 expectedStatus: HttpStatusCode.OK_200 124 expectedStatus: HttpStatusCode.OK_200
125 }) 125 })
126 126
127 expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl) 127 expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl)
128 expect(videoChannelSync.channel).to.include({ 128 expect(videoChannelSync.channel).to.include({
129 id: server.store.channel.id, 129 id: servers[0].store.channel.id,
130 name: 'root_channel' 130 name: 'root_channel'
131 }) 131 })
132 expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN) 132 expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN)
@@ -134,7 +134,7 @@ describe('Test channel synchronizations', function () {
134 }) 134 })
135 135
136 it('Should add a synchronization for another user', async function () { 136 it('Should add a synchronization for another user', async function () {
137 const { videoChannelSync } = await command.create({ 137 const { videoChannelSync } = await servers[0].channelSyncs.create({
138 attributes: { 138 attributes: {
139 externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux', 139 externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
140 videoChannelId: userInfo.channelId 140 videoChannelId: userInfo.channelId
@@ -145,9 +145,9 @@ describe('Test channel synchronizations', function () {
145 }) 145 })
146 146
147 it('Should not import a channel if not asked', async function () { 147 it('Should not import a channel if not asked', async function () {
148 await waitJobs(server) 148 await waitJobs(servers)
149 149
150 const { data } = await command.listByAccount({ accountName: userInfo.username }) 150 const { data } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })
151 151
152 expect(data[0].state).to.contain({ 152 expect(data[0].state).to.contain({
153 id: VideoChannelSyncState.WAITING_FIRST_RUN, 153 id: VideoChannelSyncState.WAITING_FIRST_RUN,
@@ -160,15 +160,15 @@ describe('Test channel synchronizations', function () {
160 160
161 await changeDateForSync(userInfo.syncId, '2019-03-01') 161 await changeDateForSync(userInfo.syncId, '2019-03-01')
162 162
163 await server.debug.sendCommand({ 163 await servers[0].debug.sendCommand({
164 body: { 164 body: {
165 command: 'process-video-channel-sync-latest' 165 command: 'process-video-channel-sync-latest'
166 } 166 }
167 }) 167 })
168 168
169 await waitJobs(server) 169 await waitJobs(servers)
170 170
171 const { data, total } = await server.videos.listByChannel({ 171 const { data, total } = await servers[0].videos.listByChannel({
172 handle: userInfo.channelName, 172 handle: userInfo.channelName,
173 include: VideoInclude.NOT_PUBLISHED_STATE 173 include: VideoInclude.NOT_PUBLISHED_STATE
174 }) 174 })
@@ -180,7 +180,7 @@ describe('Test channel synchronizations', function () {
180 it('Should list channel synchronizations', async function () { 180 it('Should list channel synchronizations', async function () {
181 // Root 181 // Root
182 { 182 {
183 const { total, data } = await command.listByAccount({ accountName: 'root' }) 183 const { total, data } = await servers[0].channelSyncs.listByAccount({ accountName: 'root' })
184 expect(total).to.equal(2) 184 expect(total).to.equal(2)
185 185
186 expect(data[0]).to.deep.contain({ 186 expect(data[0]).to.deep.contain({
@@ -193,13 +193,13 @@ describe('Test channel synchronizations', function () {
193 193
194 expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate) 194 expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate)
195 195
196 expect(data[0].channel).to.contain({ id: server.store.channel.id }) 196 expect(data[0].channel).to.contain({ id: servers[0].store.channel.id })
197 expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' }) 197 expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' })
198 } 198 }
199 199
200 // User 200 // User
201 { 201 {
202 const { total, data } = await command.listByAccount({ accountName: userInfo.username }) 202 const { total, data } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })
203 expect(total).to.equal(1) 203 expect(total).to.equal(1)
204 expect(data[0]).to.deep.contain({ 204 expect(data[0]).to.deep.contain({
205 externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux', 205 externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
@@ -212,7 +212,7 @@ describe('Test channel synchronizations', function () {
212 }) 212 })
213 213
214 it('Should list imports of a channel synchronization', async function () { 214 it('Should list imports of a channel synchronization', async function () {
215 const { total, data } = await server.imports.getMyVideoImports({ videoChannelSyncId: rootChannelSyncId }) 215 const { total, data } = await servers[0].imports.getMyVideoImports({ videoChannelSyncId: rootChannelSyncId })
216 216
217 expect(total).to.equal(1) 217 expect(total).to.equal(1)
218 expect(data).to.have.lengthOf(1) 218 expect(data).to.have.lengthOf(1)
@@ -220,14 +220,68 @@ describe('Test channel synchronizations', function () {
220 }) 220 })
221 221
222 it('Should remove user\'s channel synchronizations', async function () { 222 it('Should remove user\'s channel synchronizations', async function () {
223 await command.delete({ channelSyncId: userInfo.syncId }) 223 await servers[0].channelSyncs.delete({ channelSyncId: userInfo.syncId })
224 224
225 const { total } = await command.listByAccount({ accountName: userInfo.username }) 225 const { total } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })
226 expect(total).to.equal(0) 226 expect(total).to.equal(0)
227 }) 227 })
228 228
229 // FIXME: youtube-dl doesn't work when speicifying a port after the hostname
230 // it('Should import a remote PeerTube channel', async function () {
231 // this.timeout(240_000)
232
233 // await servers[1].videos.quickUpload({ name: 'remote 1' })
234 // await waitJobs(servers)
235
236 // const { videoChannelSync } = await servers[0].channelSyncs.create({
237 // attributes: {
238 // externalChannelUrl: servers[1].url + '/c/root_channel',
239 // videoChannelId: userInfo.channelId
240 // },
241 // token: userInfo.accessToken
242 // })
243 // await servers[0].channels.importVideos({
244 // channelName: userInfo.channelName,
245 // externalChannelUrl: servers[1].url + '/c/root_channel',
246 // videoChannelSyncId: videoChannelSync.id,
247 // token: userInfo.accessToken
248 // })
249
250 // await waitJobs(servers)
251
252 // const { data, total } = await servers[0].videos.listByChannel({
253 // handle: userInfo.channelName,
254 // include: VideoInclude.NOT_PUBLISHED_STATE
255 // })
256
257 // expect(total).to.equal(2)
258 // expect(data[0].name).to.equal('remote 1')
259 // })
260
261 // it('Should keep synced a remote PeerTube channel', async function () {
262 // this.timeout(240_000)
263
264 // await servers[1].videos.quickUpload({ name: 'remote 2' })
265 // await waitJobs(servers)
266
267 // await servers[0].debug.sendCommand({
268 // body: {
269 // command: 'process-video-channel-sync-latest'
270 // }
271 // })
272
273 // await waitJobs(servers)
274
275 // const { data, total } = await servers[0].videos.listByChannel({
276 // handle: userInfo.channelName,
277 // include: VideoInclude.NOT_PUBLISHED_STATE
278 // })
279 // expect(total).to.equal(2)
280 // expect(data[0].name).to.equal('remote 2')
281 // })
282
229 after(async function () { 283 after(async function () {
230 await server?.kill() 284 await killallServers(servers)
231 }) 285 })
232 }) 286 })
233 } 287 }