server/tests/cli/prune-storage.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import * as chai from 'chai'
   4 import { createFile, readdir } from 'fs-extra'
   5 import { join } from 'path'
   6 import { wait } from '@shared/core-utils'
   7 import { buildUUID } from '@shared/extra-utils'
   8 import { HttpStatusCode, VideoPlaylistPrivacy } from '@shared/models'
   9 import {
  10   cleanupTests,
  11   CLICommand,
  12   createMultipleServers,
  13   doubleFollow,
  14   killallServers,
  15   makeGetRequest,
  16   PeerTubeServer,
  17   setAccessTokensToServers,
  18   setDefaultVideoChannel,
  19   waitJobs
  20 } from '@shared/server-commands'
  21
  22 const expect = chai.expect
  23
  24 async function countFiles (server: PeerTubeServer, directory: string) {
  25   const files = await readdir(server.servers.buildDirectory(directory))
  26
  27   return files.length
  28 }
  29
  30 async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) {
  31   const files = await readdir(server.servers.buildDirectory(directory))
  32
  33   for (const f of files) {
  34     expect(f).to.not.contain(substring)
  35   }
  36 }
  37
  38 async function assertCountAreOkay (servers: PeerTubeServer[]) {
  39   for (const server of servers) {
  40     const videosCount = await countFiles(server, 'videos')
  41     expect(videosCount).to.equal(8)
  42
  43     const torrentsCount = await countFiles(server, 'torrents')
  44     expect(torrentsCount).to.equal(16)
  45
  46     const previewsCount = await countFiles(server, 'previews')
  47     expect(previewsCount).to.equal(2)
  48
  49     const thumbnailsCount = await countFiles(server, 'thumbnails')
  50     expect(thumbnailsCount).to.equal(6)
  51
  52     const avatarsCount = await countFiles(server, 'avatars')
  53     expect(avatarsCount).to.equal(4)
  54
  55     const hlsRootCount = await countFiles(server, 'streaming-playlists/hls')
  56     expect(hlsRootCount).to.equal(2)
  57   }
  58 }
  59
  60 describe('Test prune storage scripts', function () {
  61   let servers: PeerTubeServer[]
  62   const badNames: { [directory: string]: string[] } = {}
  63
  64   before(async function () {
  65     this.timeout(120000)
  66
  67     servers = await createMultipleServers(2, { transcoding: { enabled: true } })
  68     await setAccessTokensToServers(servers)
  69     await setDefaultVideoChannel(servers)
  70
  71     for (const server of servers) {
  72       await server.videos.upload({ attributes: { name: 'video 1' } })
  73       await server.videos.upload({ attributes: { name: 'video 2' } })
  74
  75       await server.users.updateMyAvatar({ fixture: 'avatar.png' })
  76
  77       await server.playlists.create({
  78         attributes: {
  79           displayName: 'playlist',
  80           privacy: VideoPlaylistPrivacy.PUBLIC,
  81           videoChannelId: server.store.channel.id,
  82           thumbnailfile: 'thumbnail.jpg'
  83         }
  84       })
  85     }
  86
  87     await doubleFollow(servers[0], servers[1])
  88
  89     // Lazy load the remote avatars
  90     {
  91       const account = await servers[0].accounts.get({ accountName: 'root@localhost:' + servers[1].port })
  92
  93       for (const avatar of account.avatars) {
  94         await makeGetRequest({
  95           url: servers[0].url,
  96           path: avatar.path,
  97           expectedStatus: HttpStatusCode.OK_200
  98         })
  99       }
 100     }
 101
 102     {
 103       const account = await servers[1].accounts.get({ accountName: 'root@localhost:' + servers[0].port })
 104       for (const avatar of account.avatars) {
 105         await makeGetRequest({
 106           url: servers[1].url,
 107           path: avatar.path,
 108           expectedStatus: HttpStatusCode.OK_200
 109         })
 110       }
 111     }
 112
 113     await wait(1000)
 114
 115     await waitJobs(servers)
 116     await killallServers(servers)
 117
 118     await wait(1000)
 119   })
 120
 121   it('Should have the files on the disk', async function () {
 122     await assertCountAreOkay(servers)
 123   })
 124
 125   it('Should create some dirty files', async function () {
 126     for (let i = 0; i < 2; i++) {
 127       {
 128         const base = servers[0].servers.buildDirectory('videos')
 129
 130         const n1 = buildUUID() + '.mp4'
 131         const n2 = buildUUID() + '.webm'
 132
 133         await createFile(join(base, n1))
 134         await createFile(join(base, n2))
 135
 136         badNames['videos'] = [ n1, n2 ]
 137       }
 138
 139       {
 140         const base = servers[0].servers.buildDirectory('torrents')
 141
 142         const n1 = buildUUID() + '-240.torrent'
 143         const n2 = buildUUID() + '-480.torrent'
 144
 145         await createFile(join(base, n1))
 146         await createFile(join(base, n2))
 147
 148         badNames['torrents'] = [ n1, n2 ]
 149       }
 150
 151       {
 152         const base = servers[0].servers.buildDirectory('thumbnails')
 153
 154         const n1 = buildUUID() + '.jpg'
 155         const n2 = buildUUID() + '.jpg'
 156
 157         await createFile(join(base, n1))
 158         await createFile(join(base, n2))
 159
 160         badNames['thumbnails'] = [ n1, n2 ]
 161       }
 162
 163       {
 164         const base = servers[0].servers.buildDirectory('previews')
 165
 166         const n1 = buildUUID() + '.jpg'
 167         const n2 = buildUUID() + '.jpg'
 168
 169         await createFile(join(base, n1))
 170         await createFile(join(base, n2))
 171
 172         badNames['previews'] = [ n1, n2 ]
 173       }
 174
 175       {
 176         const base = servers[0].servers.buildDirectory('avatars')
 177
 178         const n1 = buildUUID() + '.png'
 179         const n2 = buildUUID() + '.jpg'
 180
 181         await createFile(join(base, n1))
 182         await createFile(join(base, n2))
 183
 184         badNames['avatars'] = [ n1, n2 ]
 185       }
 186
 187       {
 188         const directory = join('streaming-playlists', 'hls')
 189         const base = servers[0].servers.buildDirectory(directory)
 190
 191         const n1 = buildUUID()
 192         await createFile(join(base, n1))
 193         badNames[directory] = [ n1 ]
 194       }
 195     }
 196   })
 197
 198   it('Should run prune storage', async function () {
 199     this.timeout(30000)
 200
 201     const env = servers[0].cli.getEnv()
 202     await CLICommand.exec(`echo y | ${env} npm run prune-storage`)
 203   })
 204
 205   it('Should have removed files', async function () {
 206     await assertCountAreOkay(servers)
 207
 208     for (const directory of Object.keys(badNames)) {
 209       for (const name of badNames[directory]) {
 210         await assertNotExists(servers[0], directory, name)
 211       }
 212     }
 213   })
 214
 215   after(async function () {
 216     await cleanupTests(servers)
 217   })
 218 })