server/tests/cli/prune-storage.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { createFile, readdir } from 'fs-extra'
   6 import { join } from 'path'
   7 import { buildUUID } from '@server/helpers/uuid'
   8 import { HttpStatusCode } from '@shared/core-utils'
   9 import {
  10   buildServerDirectory,
  11   cleanupTests,
  12   CLICommand,
  13   doubleFollow,
  14   flushAndRunMultipleServers,
  15   killallServers,
  16   makeGetRequest,
  17   ServerInfo,
  18   setAccessTokensToServers,
  19   setDefaultVideoChannel,
  20   updateMyAvatar,
  21   uploadVideo,
  22   wait,
  23   waitJobs
  24 } from '@shared/extra-utils'
  25 import { VideoPlaylistPrivacy } from '@shared/models'
  26
  27 const expect = chai.expect
  28
  29 async function countFiles (internalServerNumber: number, directory: string) {
  30   const files = await readdir(buildServerDirectory({ internalServerNumber }, directory))
  31
  32   return files.length
  33 }
  34
  35 async function assertNotExists (internalServerNumber: number, directory: string, substring: string) {
  36   const files = await readdir(buildServerDirectory({ internalServerNumber }, directory))
  37
  38   for (const f of files) {
  39     expect(f).to.not.contain(substring)
  40   }
  41 }
  42
  43 async function assertCountAreOkay (servers: ServerInfo[]) {
  44   for (const server of servers) {
  45     const videosCount = await countFiles(server.internalServerNumber, 'videos')
  46     expect(videosCount).to.equal(8)
  47
  48     const torrentsCount = await countFiles(server.internalServerNumber, 'torrents')
  49     expect(torrentsCount).to.equal(16)
  50
  51     const previewsCount = await countFiles(server.internalServerNumber, 'previews')
  52     expect(previewsCount).to.equal(2)
  53
  54     const thumbnailsCount = await countFiles(server.internalServerNumber, 'thumbnails')
  55     expect(thumbnailsCount).to.equal(6)
  56
  57     const avatarsCount = await countFiles(server.internalServerNumber, 'avatars')
  58     expect(avatarsCount).to.equal(2)
  59   }
  60 }
  61
  62 describe('Test prune storage scripts', function () {
  63   let servers: ServerInfo[]
  64   const badNames: { [directory: string]: string[] } = {}
  65
  66   before(async function () {
  67     this.timeout(120000)
  68
  69     servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: true } })
  70     await setAccessTokensToServers(servers)
  71     await setDefaultVideoChannel(servers)
  72
  73     for (const server of servers) {
  74       await uploadVideo(server.url, server.accessToken, { name: 'video 1' })
  75       await uploadVideo(server.url, server.accessToken, { name: 'video 2' })
  76
  77       await updateMyAvatar({ url: server.url, accessToken: server.accessToken, fixture: 'avatar.png' })
  78
  79       await server.playlistsCommand.create({
  80         attributes: {
  81           displayName: 'playlist',
  82           privacy: VideoPlaylistPrivacy.PUBLIC,
  83           videoChannelId: server.videoChannel.id,
  84           thumbnailfile: 'thumbnail.jpg'
  85         }
  86       })
  87     }
  88
  89     await doubleFollow(servers[0], servers[1])
  90
  91     // Lazy load the remote avatar
  92     {
  93       const account = await servers[0].accountsCommand.get({ accountName: 'root@localhost:' + servers[1].port })
  94       await makeGetRequest({
  95         url: servers[0].url,
  96         path: account.avatar.path,
  97         statusCodeExpected: HttpStatusCode.OK_200
  98       })
  99     }
 100
 101     {
 102       const account = await servers[1].accountsCommand.get({ accountName: 'root@localhost:' + servers[0].port })
 103       await makeGetRequest({
 104         url: servers[1].url,
 105         path: account.avatar.path,
 106         statusCodeExpected: HttpStatusCode.OK_200
 107       })
 108     }
 109
 110     await wait(1000)
 111
 112     await waitJobs(servers)
 113     killallServers(servers)
 114
 115     await wait(1000)
 116   })
 117
 118   it('Should have the files on the disk', async function () {
 119     await assertCountAreOkay(servers)
 120   })
 121
 122   it('Should create some dirty files', async function () {
 123     for (let i = 0; i < 2; i++) {
 124       {
 125         const base = buildServerDirectory(servers[0], 'videos')
 126
 127         const n1 = buildUUID() + '.mp4'
 128         const n2 = buildUUID() + '.webm'
 129
 130         await createFile(join(base, n1))
 131         await createFile(join(base, n2))
 132
 133         badNames['videos'] = [ n1, n2 ]
 134       }
 135
 136       {
 137         const base = buildServerDirectory(servers[0], 'torrents')
 138
 139         const n1 = buildUUID() + '-240.torrent'
 140         const n2 = buildUUID() + '-480.torrent'
 141
 142         await createFile(join(base, n1))
 143         await createFile(join(base, n2))
 144
 145         badNames['torrents'] = [ n1, n2 ]
 146       }
 147
 148       {
 149         const base = buildServerDirectory(servers[0], 'thumbnails')
 150
 151         const n1 = buildUUID() + '.jpg'
 152         const n2 = buildUUID() + '.jpg'
 153
 154         await createFile(join(base, n1))
 155         await createFile(join(base, n2))
 156
 157         badNames['thumbnails'] = [ n1, n2 ]
 158       }
 159
 160       {
 161         const base = buildServerDirectory(servers[0], 'previews')
 162
 163         const n1 = buildUUID() + '.jpg'
 164         const n2 = buildUUID() + '.jpg'
 165
 166         await createFile(join(base, n1))
 167         await createFile(join(base, n2))
 168
 169         badNames['previews'] = [ n1, n2 ]
 170       }
 171
 172       {
 173         const base = buildServerDirectory(servers[0], 'avatars')
 174
 175         const n1 = buildUUID() + '.png'
 176         const n2 = buildUUID() + '.jpg'
 177
 178         await createFile(join(base, n1))
 179         await createFile(join(base, n2))
 180
 181         badNames['avatars'] = [ n1, n2 ]
 182       }
 183     }
 184   })
 185
 186   it('Should run prune storage', async function () {
 187     this.timeout(30000)
 188
 189     const env = servers[0].cliCommand.getEnv()
 190     await CLICommand.exec(`echo y | ${env} npm run prune-storage`)
 191   })
 192
 193   it('Should have removed files', async function () {
 194     await assertCountAreOkay(servers)
 195
 196     for (const directory of Object.keys(badNames)) {
 197       for (const name of badNames[directory]) {
 198         await assertNotExists(servers[0].internalServerNumber, directory, name)
 199       }
 200     }
 201   })
 202
 203   after(async function () {
 204     await cleanupTests(servers)
 205   })
 206 })