server/tests/cli/prune-storage.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { createFile, readdir } from 'fs-extra'
   6 import { join } from 'path'
   7 import { wait } from '@shared/core-utils'
   8 import { buildUUID } from '@shared/extra-utils'
   9 import { HttpStatusCode, VideoPlaylistPrivacy } from '@shared/models'
  10 import {
  11   cleanupTests,
  12   CLICommand,
  13   createMultipleServers,
  14   doubleFollow,
  15   killallServers,
  16   makeGetRequest,
  17   PeerTubeServer,
  18   setAccessTokensToServers,
  19   setDefaultVideoChannel,
  20   waitJobs
  21 } from '@shared/server-commands'
  22
  23 const expect = chai.expect
  24
  25 async function countFiles (server: PeerTubeServer, directory: string) {
  26   const files = await readdir(server.servers.buildDirectory(directory))
  27
  28   return files.length
  29 }
  30
  31 async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) {
  32   const files = await readdir(server.servers.buildDirectory(directory))
  33
  34   for (const f of files) {
  35     expect(f).to.not.contain(substring)
  36   }
  37 }
  38
  39 async function assertCountAreOkay (servers: PeerTubeServer[]) {
  40   for (const server of servers) {
  41     const videosCount = await countFiles(server, 'videos')
  42     expect(videosCount).to.equal(8)
  43
  44     const torrentsCount = await countFiles(server, 'torrents')
  45     expect(torrentsCount).to.equal(16)
  46
  47     const previewsCount = await countFiles(server, 'previews')
  48     expect(previewsCount).to.equal(2)
  49
  50     const thumbnailsCount = await countFiles(server, 'thumbnails')
  51     expect(thumbnailsCount).to.equal(6)
  52
  53     const avatarsCount = await countFiles(server, 'avatars')
  54     expect(avatarsCount).to.equal(4)
  55
  56     const hlsRootCount = await countFiles(server, 'streaming-playlists/hls')
  57     expect(hlsRootCount).to.equal(2)
  58   }
  59 }
  60
  61 describe('Test prune storage scripts', function () {
  62   let servers: PeerTubeServer[]
  63   const badNames: { [directory: string]: string[] } = {}
  64
  65   before(async function () {
  66     this.timeout(120000)
  67
  68     servers = await createMultipleServers(2, { transcoding: { enabled: true } })
  69     await setAccessTokensToServers(servers)
  70     await setDefaultVideoChannel(servers)
  71
  72     for (const server of servers) {
  73       await server.videos.upload({ attributes: { name: 'video 1' } })
  74       await server.videos.upload({ attributes: { name: 'video 2' } })
  75
  76       await server.users.updateMyAvatar({ fixture: 'avatar.png' })
  77
  78       await server.playlists.create({
  79         attributes: {
  80           displayName: 'playlist',
  81           privacy: VideoPlaylistPrivacy.PUBLIC,
  82           videoChannelId: server.store.channel.id,
  83           thumbnailfile: 'thumbnail.jpg'
  84         }
  85       })
  86     }
  87
  88     await doubleFollow(servers[0], servers[1])
  89
  90     // Lazy load the remote avatars
  91     {
  92       const account = await servers[0].accounts.get({ accountName: 'root@localhost:' + servers[1].port })
  93
  94       for (const avatar of account.avatars) {
  95         await makeGetRequest({
  96           url: servers[0].url,
  97           path: avatar.path,
  98           expectedStatus: HttpStatusCode.OK_200
  99         })
 100       }
 101     }
 102
 103     {
 104       const account = await servers[1].accounts.get({ accountName: 'root@localhost:' + servers[0].port })
 105       for (const avatar of account.avatars) {
 106         await makeGetRequest({
 107           url: servers[1].url,
 108           path: avatar.path,
 109           expectedStatus: HttpStatusCode.OK_200
 110         })
 111       }
 112     }
 113
 114     await wait(1000)
 115
 116     await waitJobs(servers)
 117     await killallServers(servers)
 118
 119     await wait(1000)
 120   })
 121
 122   it('Should have the files on the disk', async function () {
 123     await assertCountAreOkay(servers)
 124   })
 125
 126   it('Should create some dirty files', async function () {
 127     for (let i = 0; i < 2; i++) {
 128       {
 129         const base = servers[0].servers.buildDirectory('videos')
 130
 131         const n1 = buildUUID() + '.mp4'
 132         const n2 = buildUUID() + '.webm'
 133
 134         await createFile(join(base, n1))
 135         await createFile(join(base, n2))
 136
 137         badNames['videos'] = [ n1, n2 ]
 138       }
 139
 140       {
 141         const base = servers[0].servers.buildDirectory('torrents')
 142
 143         const n1 = buildUUID() + '-240.torrent'
 144         const n2 = buildUUID() + '-480.torrent'
 145
 146         await createFile(join(base, n1))
 147         await createFile(join(base, n2))
 148
 149         badNames['torrents'] = [ n1, n2 ]
 150       }
 151
 152       {
 153         const base = servers[0].servers.buildDirectory('thumbnails')
 154
 155         const n1 = buildUUID() + '.jpg'
 156         const n2 = buildUUID() + '.jpg'
 157
 158         await createFile(join(base, n1))
 159         await createFile(join(base, n2))
 160
 161         badNames['thumbnails'] = [ n1, n2 ]
 162       }
 163
 164       {
 165         const base = servers[0].servers.buildDirectory('previews')
 166
 167         const n1 = buildUUID() + '.jpg'
 168         const n2 = buildUUID() + '.jpg'
 169
 170         await createFile(join(base, n1))
 171         await createFile(join(base, n2))
 172
 173         badNames['previews'] = [ n1, n2 ]
 174       }
 175
 176       {
 177         const base = servers[0].servers.buildDirectory('avatars')
 178
 179         const n1 = buildUUID() + '.png'
 180         const n2 = buildUUID() + '.jpg'
 181
 182         await createFile(join(base, n1))
 183         await createFile(join(base, n2))
 184
 185         badNames['avatars'] = [ n1, n2 ]
 186       }
 187
 188       {
 189         const directory = join('streaming-playlists', 'hls')
 190         const base = servers[0].servers.buildDirectory(directory)
 191
 192         const n1 = buildUUID()
 193         await createFile(join(base, n1))
 194         badNames[directory] = [ n1 ]
 195       }
 196     }
 197   })
 198
 199   it('Should run prune storage', async function () {
 200     this.timeout(30000)
 201
 202     const env = servers[0].cli.getEnv()
 203     await CLICommand.exec(`echo y | ${env} npm run prune-storage`)
 204   })
 205
 206   it('Should have removed files', async function () {
 207     await assertCountAreOkay(servers)
 208
 209     for (const directory of Object.keys(badNames)) {
 210       for (const name of badNames[directory]) {
 211         await assertNotExists(servers[0], directory, name)
 212       }
 213     }
 214   })
 215
 216   after(async function () {
 217     await cleanupTests(servers)
 218   })
 219 })