server/tests/api/videos/video-channel-syncs.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import { expect } from 'chai'
   4 import { FIXTURE_URLS } from '@server/tests/shared'
   5 import { areHttpImportTestsDisabled } from '@shared/core-utils'
   6 import { VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@shared/models'
   7 import {
   8   createMultipleServers,
   9   getServerImportConfig,
  10   killallServers,
  11   PeerTubeServer,
  12   setAccessTokensToServers,
  13   setDefaultAccountAvatar,
  14   setDefaultChannelAvatar,
  15   setDefaultVideoChannel,
  16   waitJobs
  17 } from '@shared/server-commands'
  18
  19 describe('Test channel synchronizations', function () {
  20   if (areHttpImportTestsDisabled()) return
  21
  22   function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
  23
  24     describe('Sync using ' + mode, function () {
  25       let servers: PeerTubeServer[]
  26
  27       let startTestDate: Date
  28
  29       let rootChannelSyncId: number
  30       const userInfo = {
  31         accessToken: '',
  32         username: 'user1',
  33         channelName: 'user1_channel',
  34         channelId: -1,
  35         syncId: -1
  36       }
  37
  38       async function changeDateForSync (channelSyncId: number, newDate: string) {
  39         await servers[0].sql.updateQuery(
  40           `UPDATE "videoChannelSync" ` +
  41           `SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` +
  42           `WHERE id=${channelSyncId}`
  43         )
  44       }
  45
  46       async function listAllVideosOfChannel (channelName: string) {
  47         return servers[0].videos.listByChannel({
  48           handle: channelName,
  49           include: VideoInclude.NOT_PUBLISHED_STATE
  50         })
  51       }
  52
  53       async function forceSyncAll (videoChannelSyncId: number, fromDate = '1970-01-01') {
  54         await changeDateForSync(videoChannelSyncId, fromDate)
  55
  56         await servers[0].debug.sendCommand({
  57           body: {
  58             command: 'process-video-channel-sync-latest'
  59           }
  60         })
  61
  62         await waitJobs(servers)
  63       }
  64
  65       before(async function () {
  66         this.timeout(240_000)
  67
  68         startTestDate = new Date()
  69
  70         servers = await createMultipleServers(2, getServerImportConfig(mode))
  71
  72         await setAccessTokensToServers(servers)
  73         await setDefaultVideoChannel(servers)
  74         await setDefaultChannelAvatar(servers)
  75         await setDefaultAccountAvatar(servers)
  76
  77         await servers[0].config.enableChannelSync()
  78
  79         {
  80           userInfo.accessToken = await servers[0].users.generateUserAndToken(userInfo.username)
  81
  82           const { videoChannels } = await servers[0].users.getMyInfo({ token: userInfo.accessToken })
  83           userInfo.channelId = videoChannels[0].id
  84         }
  85       })
  86
  87       it('Should fetch the latest channel videos of a remote channel', async function () {
  88         this.timeout(120_000)
  89
  90         {
  91           const { video } = await servers[0].imports.importVideo({
  92             attributes: {
  93               channelId: servers[0].store.channel.id,
  94               privacy: VideoPrivacy.PUBLIC,
  95               targetUrl: FIXTURE_URLS.youtube
  96             }
  97           })
  98
  99           expect(video.name).to.equal('small video - youtube')
 100           expect(video.waitTranscoding).to.be.true
 101
 102           const { total } = await listAllVideosOfChannel('root_channel')
 103           expect(total).to.equal(1)
 104         }
 105
 106         const { videoChannelSync } = await servers[0].channelSyncs.create({
 107           attributes: {
 108             externalChannelUrl: FIXTURE_URLS.youtubeChannel,
 109             videoChannelId: servers[0].store.channel.id
 110           }
 111         })
 112         rootChannelSyncId = videoChannelSync.id
 113
 114         await forceSyncAll(rootChannelSyncId)
 115
 116         {
 117           const { total, data } = await listAllVideosOfChannel('root_channel')
 118           expect(total).to.equal(2)
 119           expect(data[0].name).to.equal('test')
 120           expect(data[0].waitTranscoding).to.be.true
 121         }
 122       })
 123
 124       it('Should add another synchronization', async function () {
 125         const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar'
 126
 127         const { videoChannelSync } = await servers[0].channelSyncs.create({
 128           attributes: {
 129             externalChannelUrl,
 130             videoChannelId: servers[0].store.channel.id
 131           }
 132         })
 133
 134         expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl)
 135         expect(videoChannelSync.channel.id).to.equal(servers[0].store.channel.id)
 136         expect(videoChannelSync.channel.name).to.equal('root_channel')
 137         expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN)
 138         expect(new Date(videoChannelSync.createdAt)).to.be.above(startTestDate).and.to.be.at.most(new Date())
 139       })
 140
 141       it('Should add a synchronization for another user', async function () {
 142         const { videoChannelSync } = await servers[0].channelSyncs.create({
 143           attributes: {
 144             externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
 145             videoChannelId: userInfo.channelId
 146           },
 147           token: userInfo.accessToken
 148         })
 149         userInfo.syncId = videoChannelSync.id
 150       })
 151
 152       it('Should not import a channel if not asked', async function () {
 153         await waitJobs(servers)
 154
 155         const { data } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })
 156
 157         expect(data[0].state).to.contain({
 158           id: VideoChannelSyncState.WAITING_FIRST_RUN,
 159           label: 'Waiting first run'
 160         })
 161       })
 162
 163       it('Should only fetch the videos newer than the creation date', async function () {
 164         this.timeout(120_000)
 165
 166         await forceSyncAll(userInfo.syncId, '2019-03-01')
 167
 168         const { data, total } = await listAllVideosOfChannel(userInfo.channelName)
 169
 170         expect(total).to.equal(1)
 171         expect(data[0].name).to.equal('test')
 172       })
 173
 174       it('Should list channel synchronizations', async function () {
 175         // Root
 176         {
 177           const { total, data } = await servers[0].channelSyncs.listByAccount({ accountName: 'root' })
 178           expect(total).to.equal(2)
 179
 180           expect(data[0]).to.deep.contain({
 181             externalChannelUrl: FIXTURE_URLS.youtubeChannel,
 182             state: {
 183               id: VideoChannelSyncState.SYNCED,
 184               label: 'Synchronized'
 185             }
 186           })
 187
 188           expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate)
 189
 190           expect(data[0].channel).to.contain({ id: servers[0].store.channel.id })
 191           expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' })
 192         }
 193
 194         // User
 195         {
 196           const { total, data } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })
 197           expect(total).to.equal(1)
 198           expect(data[0]).to.deep.contain({
 199             externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
 200             state: {
 201               id: VideoChannelSyncState.SYNCED,
 202               label: 'Synchronized'
 203             }
 204           })
 205         }
 206       })
 207
 208       it('Should list imports of a channel synchronization', async function () {
 209         const { total, data } = await servers[0].imports.getMyVideoImports({ videoChannelSyncId: rootChannelSyncId })
 210
 211         expect(total).to.equal(1)
 212         expect(data).to.have.lengthOf(1)
 213         expect(data[0].video.name).to.equal('test')
 214       })
 215
 216       it('Should remove user\'s channel synchronizations', async function () {
 217         await servers[0].channelSyncs.delete({ channelSyncId: userInfo.syncId })
 218
 219         const { total } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })
 220         expect(total).to.equal(0)
 221       })
 222
 223       // FIXME: youtube-dl/yt-dlp doesn't work when speicifying a port after the hostname
 224       // it('Should import a remote PeerTube channel', async function () {
 225       //   this.timeout(240_000)
 226
 227       //   await servers[1].videos.quickUpload({ name: 'remote 1' })
 228       //   await waitJobs(servers)
 229
 230       //   const { videoChannelSync } = await servers[0].channelSyncs.create({
 231       //     attributes: {
 232       //       externalChannelUrl: servers[1].url + '/c/root_channel',
 233       //       videoChannelId: userInfo.channelId
 234       //     },
 235       //     token: userInfo.accessToken
 236       //   })
 237       //   await servers[0].channels.importVideos({
 238       //     channelName: userInfo.channelName,
 239       //     externalChannelUrl: servers[1].url + '/c/root_channel',
 240       //     videoChannelSyncId: videoChannelSync.id,
 241       //     token: userInfo.accessToken
 242       //   })
 243
 244       //   await waitJobs(servers)
 245
 246       //   const { data, total } = await servers[0].videos.listByChannel({
 247       //     handle: userInfo.channelName,
 248       //     include: VideoInclude.NOT_PUBLISHED_STATE
 249       //   })
 250
 251       //   expect(total).to.equal(2)
 252       //   expect(data[0].name).to.equal('remote 1')
 253       // })
 254
 255       // it('Should keep synced a remote PeerTube channel', async function () {
 256       //   this.timeout(240_000)
 257
 258       //   await servers[1].videos.quickUpload({ name: 'remote 2' })
 259       //   await waitJobs(servers)
 260
 261       //   await servers[0].debug.sendCommand({
 262       //     body: {
 263       //       command: 'process-video-channel-sync-latest'
 264       //     }
 265       //   })
 266
 267       //   await waitJobs(servers)
 268
 269       //   const { data, total } = await servers[0].videos.listByChannel({
 270       //     handle: userInfo.channelName,
 271       //     include: VideoInclude.NOT_PUBLISHED_STATE
 272       //   })
 273       //   expect(total).to.equal(2)
 274       //   expect(data[0].name).to.equal('remote 2')
 275       // })
 276
 277       it('Should fetch the latest videos of a youtube playlist', async function () {
 278         this.timeout(120_000)
 279
 280         const { id: channelId } = await servers[0].channels.create({
 281           attributes: {
 282             name: 'channel2'
 283           }
 284         })
 285
 286         const { videoChannelSync: { id: videoChannelSyncId } } = await servers[0].channelSyncs.create({
 287           attributes: {
 288             externalChannelUrl: FIXTURE_URLS.youtubePlaylist,
 289             videoChannelId: channelId
 290           }
 291         })
 292
 293         await forceSyncAll(videoChannelSyncId)
 294
 295         {
 296
 297           const { total, data } = await listAllVideosOfChannel('channel2')
 298           expect(total).to.equal(2)
 299           expect(data[0].name).to.equal('test')
 300           expect(data[1].name).to.equal('small video - youtube')
 301         }
 302       })
 303
 304       after(async function () {
 305         await killallServers(servers)
 306       })
 307     })
 308   }
 309
 310   runSuite('youtube-dl')
 311   runSuite('yt-dlp')
 312 })