server/tests/api/videos/video-channel-syncs.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import { expect } from 'chai'
   4 import { FIXTURE_URLS, SQLCommand } from '@server/tests/shared'
   5 import { areHttpImportTestsDisabled } from '@shared/core-utils'
   6 import { VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@shared/models'
   7 import {
   8   cleanupTests,
   9   createMultipleServers,
  10   getServerImportConfig,
  11   PeerTubeServer,
  12   setAccessTokensToServers,
  13   setDefaultAccountAvatar,
  14   setDefaultChannelAvatar,
  15   setDefaultVideoChannel,
  16   waitJobs
  17 } from '@shared/server-commands'
  18
  19 describe('Test channel synchronizations', function () {
  20   if (areHttpImportTestsDisabled()) return
  21
  22   function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
  23
  24     describe('Sync using ' + mode, function () {
  25       let servers: PeerTubeServer[]
  26       let sqlCommands: SQLCommand[]
  27
  28       let startTestDate: Date
  29
  30       let rootChannelSyncId: number
  31       const userInfo = {
  32         accessToken: '',
  33         username: 'user1',
  34         channelName: 'user1_channel',
  35         channelId: -1,
  36         syncId: -1
  37       }
  38
  39       async function changeDateForSync (channelSyncId: number, newDate: string) {
  40         await sqlCommands[0].updateQuery(
  41           `UPDATE "videoChannelSync" ` +
  42           `SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` +
  43           `WHERE id=${channelSyncId}`
  44         )
  45       }
  46
  47       async function listAllVideosOfChannel (channelName: string) {
  48         return servers[0].videos.listByChannel({
  49           handle: channelName,
  50           include: VideoInclude.NOT_PUBLISHED_STATE
  51         })
  52       }
  53
  54       async function forceSyncAll (videoChannelSyncId: number, fromDate = '1970-01-01') {
  55         await changeDateForSync(videoChannelSyncId, fromDate)
  56
  57         await servers[0].debug.sendCommand({
  58           body: {
  59             command: 'process-video-channel-sync-latest'
  60           }
  61         })
  62
  63         await waitJobs(servers)
  64       }
  65
  66       before(async function () {
  67         this.timeout(240_000)
  68
  69         startTestDate = new Date()
  70
  71         servers = await createMultipleServers(2, getServerImportConfig(mode))
  72
  73         await setAccessTokensToServers(servers)
  74         await setDefaultVideoChannel(servers)
  75         await setDefaultChannelAvatar(servers)
  76         await setDefaultAccountAvatar(servers)
  77
  78         await servers[0].config.enableChannelSync()
  79
  80         {
  81           userInfo.accessToken = await servers[0].users.generateUserAndToken(userInfo.username)
  82
  83           const { videoChannels } = await servers[0].users.getMyInfo({ token: userInfo.accessToken })
  84           userInfo.channelId = videoChannels[0].id
  85         }
  86
  87         sqlCommands = servers.map(s => new SQLCommand(s))
  88       })
  89
  90       it('Should fetch the latest channel videos of a remote channel', async function () {
  91         this.timeout(120_000)
  92
  93         {
  94           const { video } = await servers[0].imports.importVideo({
  95             attributes: {
  96               channelId: servers[0].store.channel.id,
  97               privacy: VideoPrivacy.PUBLIC,
  98               targetUrl: FIXTURE_URLS.youtube
  99             }
 100           })
 101
 102           expect(video.name).to.equal('small video - youtube')
 103           expect(video.waitTranscoding).to.be.true
 104
 105           const { total } = await listAllVideosOfChannel('root_channel')
 106           expect(total).to.equal(1)
 107         }
 108
 109         const { videoChannelSync } = await servers[0].channelSyncs.create({
 110           attributes: {
 111             externalChannelUrl: FIXTURE_URLS.youtubeChannel,
 112             videoChannelId: servers[0].store.channel.id
 113           }
 114         })
 115         rootChannelSyncId = videoChannelSync.id
 116
 117         await forceSyncAll(rootChannelSyncId)
 118
 119         {
 120           const { total, data } = await listAllVideosOfChannel('root_channel')
 121           expect(total).to.equal(2)
 122           expect(data[0].name).to.equal('test')
 123           expect(data[0].waitTranscoding).to.be.true
 124         }
 125       })
 126
 127       it('Should add another synchronization', async function () {
 128         const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar'
 129
 130         const { videoChannelSync } = await servers[0].channelSyncs.create({
 131           attributes: {
 132             externalChannelUrl,
 133             videoChannelId: servers[0].store.channel.id
 134           }
 135         })
 136
 137         expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl)
 138         expect(videoChannelSync.channel.id).to.equal(servers[0].store.channel.id)
 139         expect(videoChannelSync.channel.name).to.equal('root_channel')
 140         expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN)
 141         expect(new Date(videoChannelSync.createdAt)).to.be.above(startTestDate).and.to.be.at.most(new Date())
 142       })
 143
 144       it('Should add a synchronization for another user', async function () {
 145         const { videoChannelSync } = await servers[0].channelSyncs.create({
 146           attributes: {
 147             externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
 148             videoChannelId: userInfo.channelId
 149           },
 150           token: userInfo.accessToken
 151         })
 152         userInfo.syncId = videoChannelSync.id
 153       })
 154
 155       it('Should not import a channel if not asked', async function () {
 156         await waitJobs(servers)
 157
 158         const { data } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })
 159
 160         expect(data[0].state).to.contain({
 161           id: VideoChannelSyncState.WAITING_FIRST_RUN,
 162           label: 'Waiting first run'
 163         })
 164       })
 165
 166       it('Should only fetch the videos newer than the creation date', async function () {
 167         this.timeout(120_000)
 168
 169         await forceSyncAll(userInfo.syncId, '2019-03-01')
 170
 171         const { data, total } = await listAllVideosOfChannel(userInfo.channelName)
 172
 173         expect(total).to.equal(1)
 174         expect(data[0].name).to.equal('test')
 175       })
 176
 177       it('Should list channel synchronizations', async function () {
 178         // Root
 179         {
 180           const { total, data } = await servers[0].channelSyncs.listByAccount({ accountName: 'root' })
 181           expect(total).to.equal(2)
 182
 183           expect(data[0]).to.deep.contain({
 184             externalChannelUrl: FIXTURE_URLS.youtubeChannel,
 185             state: {
 186               id: VideoChannelSyncState.SYNCED,
 187               label: 'Synchronized'
 188             }
 189           })
 190
 191           expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate)
 192
 193           expect(data[0].channel).to.contain({ id: servers[0].store.channel.id })
 194           expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' })
 195         }
 196
 197         // User
 198         {
 199           const { total, data } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })
 200           expect(total).to.equal(1)
 201           expect(data[0]).to.deep.contain({
 202             externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
 203             state: {
 204               id: VideoChannelSyncState.SYNCED,
 205               label: 'Synchronized'
 206             }
 207           })
 208         }
 209       })
 210
 211       it('Should list imports of a channel synchronization', async function () {
 212         const { total, data } = await servers[0].imports.getMyVideoImports({ videoChannelSyncId: rootChannelSyncId })
 213
 214         expect(total).to.equal(1)
 215         expect(data).to.have.lengthOf(1)
 216         expect(data[0].video.name).to.equal('test')
 217       })
 218
 219       it('Should remove user\'s channel synchronizations', async function () {
 220         await servers[0].channelSyncs.delete({ channelSyncId: userInfo.syncId })
 221
 222         const { total } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })
 223         expect(total).to.equal(0)
 224       })
 225
 226       // FIXME: youtube-dl/yt-dlp doesn't work when speicifying a port after the hostname
 227       // it('Should import a remote PeerTube channel', async function () {
 228       //   this.timeout(240_000)
 229
 230       //   await servers[1].videos.quickUpload({ name: 'remote 1' })
 231       //   await waitJobs(servers)
 232
 233       //   const { videoChannelSync } = await servers[0].channelSyncs.create({
 234       //     attributes: {
 235       //       externalChannelUrl: servers[1].url + '/c/root_channel',
 236       //       videoChannelId: userInfo.channelId
 237       //     },
 238       //     token: userInfo.accessToken
 239       //   })
 240       //   await servers[0].channels.importVideos({
 241       //     channelName: userInfo.channelName,
 242       //     externalChannelUrl: servers[1].url + '/c/root_channel',
 243       //     videoChannelSyncId: videoChannelSync.id,
 244       //     token: userInfo.accessToken
 245       //   })
 246
 247       //   await waitJobs(servers)
 248
 249       //   const { data, total } = await servers[0].videos.listByChannel({
 250       //     handle: userInfo.channelName,
 251       //     include: VideoInclude.NOT_PUBLISHED_STATE
 252       //   })
 253
 254       //   expect(total).to.equal(2)
 255       //   expect(data[0].name).to.equal('remote 1')
 256       // })
 257
 258       // it('Should keep synced a remote PeerTube channel', async function () {
 259       //   this.timeout(240_000)
 260
 261       //   await servers[1].videos.quickUpload({ name: 'remote 2' })
 262       //   await waitJobs(servers)
 263
 264       //   await servers[0].debug.sendCommand({
 265       //     body: {
 266       //       command: 'process-video-channel-sync-latest'
 267       //     }
 268       //   })
 269
 270       //   await waitJobs(servers)
 271
 272       //   const { data, total } = await servers[0].videos.listByChannel({
 273       //     handle: userInfo.channelName,
 274       //     include: VideoInclude.NOT_PUBLISHED_STATE
 275       //   })
 276       //   expect(total).to.equal(2)
 277       //   expect(data[0].name).to.equal('remote 2')
 278       // })
 279
 280       it('Should fetch the latest videos of a youtube playlist', async function () {
 281         this.timeout(120_000)
 282
 283         const { id: channelId } = await servers[0].channels.create({
 284           attributes: {
 285             name: 'channel2'
 286           }
 287         })
 288
 289         const { videoChannelSync: { id: videoChannelSyncId } } = await servers[0].channelSyncs.create({
 290           attributes: {
 291             externalChannelUrl: FIXTURE_URLS.youtubePlaylist,
 292             videoChannelId: channelId
 293           }
 294         })
 295
 296         await forceSyncAll(videoChannelSyncId)
 297
 298         {
 299
 300           const { total, data } = await listAllVideosOfChannel('channel2')
 301           expect(total).to.equal(2)
 302           expect(data[0].name).to.equal('test')
 303           expect(data[1].name).to.equal('small video - youtube')
 304         }
 305       })
 306
 307       after(async function () {
 308         for (const sqlCommand of sqlCommands) {
 309           await sqlCommand.cleanup()
 310         }
 311
 312         await cleanupTests(servers)
 313       })
 314     })
 315   }
 316
 317   // FIXME: suite is broken with youtube-dl
 318   // runSuite('youtube-dl')
 319   runSuite('yt-dlp')
 320 })