server/tests/cli/prune-storage.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import { expect } from 'chai'
   4 import { createFile, readdir } from 'fs-extra'
   5 import { join } from 'path'
   6 import { wait } from '@shared/core-utils'
   7 import { buildUUID } from '@shared/extra-utils'
   8 import { HttpStatusCode, VideoPlaylistPrivacy } from '@shared/models'
   9 import {
  10   cleanupTests,
  11   CLICommand,
  12   createMultipleServers,
  13   doubleFollow,
  14   killallServers,
  15   makeGetRequest,
  16   PeerTubeServer,
  17   setAccessTokensToServers,
  18   setDefaultVideoChannel,
  19   waitJobs
  20 } from '@shared/server-commands'
  21
  22 async function countFiles (server: PeerTubeServer, directory: string) {
  23   const files = await readdir(server.servers.buildDirectory(directory))
  24
  25   return files.length
  26 }
  27
  28 async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) {
  29   const files = await readdir(server.servers.buildDirectory(directory))
  30
  31   for (const f of files) {
  32     expect(f).to.not.contain(substring)
  33   }
  34 }
  35
  36 async function assertCountAreOkay (servers: PeerTubeServer[]) {
  37   for (const server of servers) {
  38     const videosCount = await countFiles(server, 'videos')
  39     expect(videosCount).to.equal(8)
  40
  41     const torrentsCount = await countFiles(server, 'torrents')
  42     expect(torrentsCount).to.equal(16)
  43
  44     const previewsCount = await countFiles(server, 'previews')
  45     expect(previewsCount).to.equal(2)
  46
  47     const thumbnailsCount = await countFiles(server, 'thumbnails')
  48     expect(thumbnailsCount).to.equal(6)
  49
  50     const avatarsCount = await countFiles(server, 'avatars')
  51     expect(avatarsCount).to.equal(4)
  52
  53     const hlsRootCount = await countFiles(server, 'streaming-playlists/hls')
  54     expect(hlsRootCount).to.equal(2)
  55   }
  56 }
  57
  58 describe('Test prune storage scripts', function () {
  59   let servers: PeerTubeServer[]
  60   const badNames: { [directory: string]: string[] } = {}
  61
  62   before(async function () {
  63     this.timeout(120000)
  64
  65     servers = await createMultipleServers(2, { transcoding: { enabled: true } })
  66     await setAccessTokensToServers(servers)
  67     await setDefaultVideoChannel(servers)
  68
  69     for (const server of servers) {
  70       await server.videos.upload({ attributes: { name: 'video 1' } })
  71       await server.videos.upload({ attributes: { name: 'video 2' } })
  72
  73       await server.users.updateMyAvatar({ fixture: 'avatar.png' })
  74
  75       await server.playlists.create({
  76         attributes: {
  77           displayName: 'playlist',
  78           privacy: VideoPlaylistPrivacy.PUBLIC,
  79           videoChannelId: server.store.channel.id,
  80           thumbnailfile: 'thumbnail.jpg'
  81         }
  82       })
  83     }
  84
  85     await doubleFollow(servers[0], servers[1])
  86
  87     // Lazy load the remote avatars
  88     {
  89       const account = await servers[0].accounts.get({ accountName: 'root@localhost:' + servers[1].port })
  90
  91       for (const avatar of account.avatars) {
  92         await makeGetRequest({
  93           url: servers[0].url,
  94           path: avatar.path,
  95           expectedStatus: HttpStatusCode.OK_200
  96         })
  97       }
  98     }
  99
 100     {
 101       const account = await servers[1].accounts.get({ accountName: 'root@localhost:' + servers[0].port })
 102       for (const avatar of account.avatars) {
 103         await makeGetRequest({
 104           url: servers[1].url,
 105           path: avatar.path,
 106           expectedStatus: HttpStatusCode.OK_200
 107         })
 108       }
 109     }
 110
 111     await wait(1000)
 112
 113     await waitJobs(servers)
 114     await killallServers(servers)
 115
 116     await wait(1000)
 117   })
 118
 119   it('Should have the files on the disk', async function () {
 120     await assertCountAreOkay(servers)
 121   })
 122
 123   it('Should create some dirty files', async function () {
 124     for (let i = 0; i < 2; i++) {
 125       {
 126         const base = servers[0].servers.buildDirectory('videos')
 127
 128         const n1 = buildUUID() + '.mp4'
 129         const n2 = buildUUID() + '.webm'
 130
 131         await createFile(join(base, n1))
 132         await createFile(join(base, n2))
 133
 134         badNames['videos'] = [ n1, n2 ]
 135       }
 136
 137       {
 138         const base = servers[0].servers.buildDirectory('torrents')
 139
 140         const n1 = buildUUID() + '-240.torrent'
 141         const n2 = buildUUID() + '-480.torrent'
 142
 143         await createFile(join(base, n1))
 144         await createFile(join(base, n2))
 145
 146         badNames['torrents'] = [ n1, n2 ]
 147       }
 148
 149       {
 150         const base = servers[0].servers.buildDirectory('thumbnails')
 151
 152         const n1 = buildUUID() + '.jpg'
 153         const n2 = buildUUID() + '.jpg'
 154
 155         await createFile(join(base, n1))
 156         await createFile(join(base, n2))
 157
 158         badNames['thumbnails'] = [ n1, n2 ]
 159       }
 160
 161       {
 162         const base = servers[0].servers.buildDirectory('previews')
 163
 164         const n1 = buildUUID() + '.jpg'
 165         const n2 = buildUUID() + '.jpg'
 166
 167         await createFile(join(base, n1))
 168         await createFile(join(base, n2))
 169
 170         badNames['previews'] = [ n1, n2 ]
 171       }
 172
 173       {
 174         const base = servers[0].servers.buildDirectory('avatars')
 175
 176         const n1 = buildUUID() + '.png'
 177         const n2 = buildUUID() + '.jpg'
 178
 179         await createFile(join(base, n1))
 180         await createFile(join(base, n2))
 181
 182         badNames['avatars'] = [ n1, n2 ]
 183       }
 184
 185       {
 186         const directory = join('streaming-playlists', 'hls')
 187         const base = servers[0].servers.buildDirectory(directory)
 188
 189         const n1 = buildUUID()
 190         await createFile(join(base, n1))
 191         badNames[directory] = [ n1 ]
 192       }
 193     }
 194   })
 195
 196   it('Should run prune storage', async function () {
 197     this.timeout(30000)
 198
 199     const env = servers[0].cli.getEnv()
 200     await CLICommand.exec(`echo y | ${env} npm run prune-storage`)
 201   })
 202
 203   it('Should have removed files', async function () {
 204     await assertCountAreOkay(servers)
 205
 206     for (const directory of Object.keys(badNames)) {
 207       for (const name of badNames[directory]) {
 208         await assertNotExists(servers[0], directory, name)
 209       }
 210     }
 211   })
 212
 213   after(async function () {
 214     await cleanupTests(servers)
 215   })
 216 })