server/tests/cli/prune-storage.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { createFile, readdir } from 'fs-extra'
   6 import { join } from 'path'
   7 import { buildUUID } from '@server/helpers/uuid'
   8 import {
   9   cleanupTests,
  10   CLICommand,
  11   createMultipleServers,
  12   doubleFollow,
  13   killallServers,
  14   makeGetRequest,
  15   PeerTubeServer,
  16   setAccessTokensToServers,
  17   setDefaultVideoChannel,
  18   wait,
  19   waitJobs
  20 } from '@shared/extra-utils'
  21 import { HttpStatusCode, VideoPlaylistPrivacy } from '@shared/models'
  22
  23 const expect = chai.expect
  24
  25 async function countFiles (server: PeerTubeServer, directory: string) {
  26   const files = await readdir(server.servers.buildDirectory(directory))
  27
  28   return files.length
  29 }
  30
  31 async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) {
  32   const files = await readdir(server.servers.buildDirectory(directory))
  33
  34   for (const f of files) {
  35     expect(f).to.not.contain(substring)
  36   }
  37 }
  38
  39 async function assertCountAreOkay (servers: PeerTubeServer[], videoServer2UUID: string) {
  40   for (const server of servers) {
  41     const videosCount = await countFiles(server, 'videos')
  42     expect(videosCount).to.equal(8)
  43
  44     const torrentsCount = await countFiles(server, 'torrents')
  45     expect(torrentsCount).to.equal(16)
  46
  47     const previewsCount = await countFiles(server, 'previews')
  48     expect(previewsCount).to.equal(2)
  49
  50     const thumbnailsCount = await countFiles(server, 'thumbnails')
  51     expect(thumbnailsCount).to.equal(6)
  52
  53     const avatarsCount = await countFiles(server, 'avatars')
  54     expect(avatarsCount).to.equal(2)
  55   }
  56
  57   // When we'll prune HLS directories too
  58   // const hlsRootCount = await countFiles(servers[1], 'streaming-playlists/hls/')
  59   // expect(hlsRootCount).to.equal(2)
  60
  61   // const hlsCount = await countFiles(servers[1], 'streaming-playlists/hls/' + videoServer2UUID)
  62   // expect(hlsCount).to.equal(10)
  63 }
  64
  65 describe('Test prune storage scripts', function () {
  66   let servers: PeerTubeServer[]
  67   const badNames: { [directory: string]: string[] } = {}
  68
  69   let videoServer2UUID: string
  70
  71   before(async function () {
  72     this.timeout(120000)
  73
  74     servers = await createMultipleServers(2, { transcoding: { enabled: true } })
  75     await setAccessTokensToServers(servers)
  76     await setDefaultVideoChannel(servers)
  77
  78     for (const server of servers) {
  79       await server.videos.upload({ attributes: { name: 'video 1' } })
  80
  81       const { uuid } = await server.videos.upload({ attributes: { name: 'video 2' } })
  82       if (server.serverNumber === 2) videoServer2UUID = uuid
  83
  84       await server.users.updateMyAvatar({ fixture: 'avatar.png' })
  85
  86       await server.playlists.create({
  87         attributes: {
  88           displayName: 'playlist',
  89           privacy: VideoPlaylistPrivacy.PUBLIC,
  90           videoChannelId: server.store.channel.id,
  91           thumbnailfile: 'thumbnail.jpg'
  92         }
  93       })
  94     }
  95
  96     await doubleFollow(servers[0], servers[1])
  97
  98     // Lazy load the remote avatar
  99     {
 100       const account = await servers[0].accounts.get({ accountName: 'root@localhost:' + servers[1].port })
 101       await makeGetRequest({
 102         url: servers[0].url,
 103         path: account.avatar.path,
 104         expectedStatus: HttpStatusCode.OK_200
 105       })
 106     }
 107
 108     {
 109       const account = await servers[1].accounts.get({ accountName: 'root@localhost:' + servers[0].port })
 110       await makeGetRequest({
 111         url: servers[1].url,
 112         path: account.avatar.path,
 113         expectedStatus: HttpStatusCode.OK_200
 114       })
 115     }
 116
 117     await wait(1000)
 118
 119     await waitJobs(servers)
 120     await killallServers(servers)
 121
 122     await wait(1000)
 123   })
 124
 125   it('Should have the files on the disk', async function () {
 126     await assertCountAreOkay(servers, videoServer2UUID)
 127   })
 128
 129   it('Should create some dirty files', async function () {
 130     for (let i = 0; i < 2; i++) {
 131       {
 132         const base = servers[0].servers.buildDirectory('videos')
 133
 134         const n1 = buildUUID() + '.mp4'
 135         const n2 = buildUUID() + '.webm'
 136
 137         await createFile(join(base, n1))
 138         await createFile(join(base, n2))
 139
 140         badNames['videos'] = [ n1, n2 ]
 141       }
 142
 143       {
 144         const base = servers[0].servers.buildDirectory('torrents')
 145
 146         const n1 = buildUUID() + '-240.torrent'
 147         const n2 = buildUUID() + '-480.torrent'
 148
 149         await createFile(join(base, n1))
 150         await createFile(join(base, n2))
 151
 152         badNames['torrents'] = [ n1, n2 ]
 153       }
 154
 155       {
 156         const base = servers[0].servers.buildDirectory('thumbnails')
 157
 158         const n1 = buildUUID() + '.jpg'
 159         const n2 = buildUUID() + '.jpg'
 160
 161         await createFile(join(base, n1))
 162         await createFile(join(base, n2))
 163
 164         badNames['thumbnails'] = [ n1, n2 ]
 165       }
 166
 167       {
 168         const base = servers[0].servers.buildDirectory('previews')
 169
 170         const n1 = buildUUID() + '.jpg'
 171         const n2 = buildUUID() + '.jpg'
 172
 173         await createFile(join(base, n1))
 174         await createFile(join(base, n2))
 175
 176         badNames['previews'] = [ n1, n2 ]
 177       }
 178
 179       {
 180         const base = servers[0].servers.buildDirectory('avatars')
 181
 182         const n1 = buildUUID() + '.png'
 183         const n2 = buildUUID() + '.jpg'
 184
 185         await createFile(join(base, n1))
 186         await createFile(join(base, n2))
 187
 188         badNames['avatars'] = [ n1, n2 ]
 189       }
 190
 191       // When we'll prune HLS directories too
 192       // {
 193       //   const directory = join('streaming-playlists', 'hls')
 194       //   const base = servers[1].servers.buildDirectory(directory)
 195
 196       //   const n1 = buildUUID()
 197       //   await createFile(join(base, n1))
 198       //   badNames[directory] = [ n1 ]
 199       // }
 200
 201       // {
 202       //   const directory = join('streaming-playlists', 'hls', videoServer2UUID)
 203       //   const base = servers[1].servers.buildDirectory(directory)
 204       //   const n1 = buildUUID() + '-240-fragmented-.mp4'
 205       //   const n2 = buildUUID() + '-master.m3u8'
 206
 207       //   await createFile(join(base, n1))
 208       //   await createFile(join(base, n2))
 209
 210       //   badNames[directory] = [ n1, n2 ]
 211       // }
 212     }
 213   })
 214
 215   it('Should run prune storage', async function () {
 216     this.timeout(30000)
 217
 218     const env = servers[0].cli.getEnv()
 219     await CLICommand.exec(`echo y | ${env} npm run prune-storage`)
 220   })
 221
 222   it('Should have removed files', async function () {
 223     await assertCountAreOkay(servers, videoServer2UUID)
 224
 225     for (const directory of Object.keys(badNames)) {
 226       for (const name of badNames[directory]) {
 227         await assertNotExists(servers[0], directory, name)
 228       }
 229     }
 230   })
 231
 232   after(async function () {
 233     await cleanupTests(servers)
 234   })
 235 })