server/tests/cli/prune-storage.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { createFile, readdir } from 'fs-extra'
   6 import { join } from 'path'
   7 import { buildUUID } from '@server/helpers/uuid'
   8 import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
   9 import {
  10   buildServerDirectory,
  11   cleanupTests,
  12   createVideoPlaylist,
  13   doubleFollow,
  14   execCLI,
  15   flushAndRunMultipleServers,
  16   getAccount,
  17   getEnvCli,
  18   killallServers,
  19   makeGetRequest,
  20   ServerInfo,
  21   setAccessTokensToServers,
  22   setDefaultVideoChannel,
  23   updateMyAvatar,
  24   uploadVideo,
  25   wait
  26 } from '../../../shared/extra-utils'
  27 import { waitJobs } from '../../../shared/extra-utils/server/jobs'
  28 import { Account, VideoPlaylistPrivacy } from '../../../shared/models'
  29
  30 const expect = chai.expect
  31
  32 async function countFiles (internalServerNumber: number, directory: string) {
  33   const files = await readdir(buildServerDirectory({ internalServerNumber }, directory))
  34
  35   return files.length
  36 }
  37
  38 async function assertNotExists (internalServerNumber: number, directory: string, substring: string) {
  39   const files = await readdir(buildServerDirectory({ internalServerNumber }, directory))
  40
  41   for (const f of files) {
  42     expect(f).to.not.contain(substring)
  43   }
  44 }
  45
  46 async function assertCountAreOkay (servers: ServerInfo[]) {
  47   for (const server of servers) {
  48     const videosCount = await countFiles(server.internalServerNumber, 'videos')
  49     expect(videosCount).to.equal(8)
  50
  51     const torrentsCount = await countFiles(server.internalServerNumber, 'torrents')
  52     expect(torrentsCount).to.equal(16)
  53
  54     const previewsCount = await countFiles(server.internalServerNumber, 'previews')
  55     expect(previewsCount).to.equal(2)
  56
  57     const thumbnailsCount = await countFiles(server.internalServerNumber, 'thumbnails')
  58     expect(thumbnailsCount).to.equal(6)
  59
  60     const avatarsCount = await countFiles(server.internalServerNumber, 'avatars')
  61     expect(avatarsCount).to.equal(2)
  62   }
  63 }
  64
  65 describe('Test prune storage scripts', function () {
  66   let servers: ServerInfo[]
  67   const badNames: { [directory: string]: string[] } = {}
  68
  69   before(async function () {
  70     this.timeout(120000)
  71
  72     servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: true } })
  73     await setAccessTokensToServers(servers)
  74     await setDefaultVideoChannel(servers)
  75
  76     for (const server of servers) {
  77       await uploadVideo(server.url, server.accessToken, { name: 'video 1' })
  78       await uploadVideo(server.url, server.accessToken, { name: 'video 2' })
  79
  80       await updateMyAvatar({ url: server.url, accessToken: server.accessToken, fixture: 'avatar.png' })
  81
  82       await createVideoPlaylist({
  83         url: server.url,
  84         token: server.accessToken,
  85         playlistAttrs: {
  86           displayName: 'playlist',
  87           privacy: VideoPlaylistPrivacy.PUBLIC,
  88           videoChannelId: server.videoChannel.id,
  89           thumbnailfile: 'thumbnail.jpg'
  90         }
  91       })
  92     }
  93
  94     await doubleFollow(servers[0], servers[1])
  95
  96     // Lazy load the remote avatar
  97     {
  98       const res = await getAccount(servers[0].url, 'root@localhost:' + servers[1].port)
  99       const account: Account = res.body
 100       await makeGetRequest({
 101         url: servers[0].url,
 102         path: account.avatar.path,
 103         statusCodeExpected: HttpStatusCode.OK_200
 104       })
 105     }
 106
 107     {
 108       const res = await getAccount(servers[1].url, 'root@localhost:' + servers[0].port)
 109       const account: Account = res.body
 110       await makeGetRequest({
 111         url: servers[1].url,
 112         path: account.avatar.path,
 113         statusCodeExpected: HttpStatusCode.OK_200
 114       })
 115     }
 116
 117     await wait(1000)
 118
 119     await waitJobs(servers)
 120     killallServers(servers)
 121
 122     await wait(1000)
 123   })
 124
 125   it('Should have the files on the disk', async function () {
 126     await assertCountAreOkay(servers)
 127   })
 128
 129   it('Should create some dirty files', async function () {
 130     for (let i = 0; i < 2; i++) {
 131       {
 132         const base = buildServerDirectory(servers[0], 'videos')
 133
 134         const n1 = buildUUID() + '.mp4'
 135         const n2 = buildUUID() + '.webm'
 136
 137         await createFile(join(base, n1))
 138         await createFile(join(base, n2))
 139
 140         badNames['videos'] = [ n1, n2 ]
 141       }
 142
 143       {
 144         const base = buildServerDirectory(servers[0], 'torrents')
 145
 146         const n1 = buildUUID() + '-240.torrent'
 147         const n2 = buildUUID() + '-480.torrent'
 148
 149         await createFile(join(base, n1))
 150         await createFile(join(base, n2))
 151
 152         badNames['torrents'] = [ n1, n2 ]
 153       }
 154
 155       {
 156         const base = buildServerDirectory(servers[0], 'thumbnails')
 157
 158         const n1 = buildUUID() + '.jpg'
 159         const n2 = buildUUID() + '.jpg'
 160
 161         await createFile(join(base, n1))
 162         await createFile(join(base, n2))
 163
 164         badNames['thumbnails'] = [ n1, n2 ]
 165       }
 166
 167       {
 168         const base = buildServerDirectory(servers[0], 'previews')
 169
 170         const n1 = buildUUID() + '.jpg'
 171         const n2 = buildUUID() + '.jpg'
 172
 173         await createFile(join(base, n1))
 174         await createFile(join(base, n2))
 175
 176         badNames['previews'] = [ n1, n2 ]
 177       }
 178
 179       {
 180         const base = buildServerDirectory(servers[0], 'avatars')
 181
 182         const n1 = buildUUID() + '.png'
 183         const n2 = buildUUID() + '.jpg'
 184
 185         await createFile(join(base, n1))
 186         await createFile(join(base, n2))
 187
 188         badNames['avatars'] = [ n1, n2 ]
 189       }
 190     }
 191   })
 192
 193   it('Should run prune storage', async function () {
 194     this.timeout(30000)
 195
 196     const env = getEnvCli(servers[0])
 197     await execCLI(`echo y | ${env} npm run prune-storage`)
 198   })
 199
 200   it('Should have removed files', async function () {
 201     await assertCountAreOkay(servers)
 202
 203     for (const directory of Object.keys(badNames)) {
 204       for (const name of badNames[directory]) {
 205         await assertNotExists(servers[0].internalServerNumber, directory, name)
 206       }
 207     }
 208   })
 209
 210   after(async function () {
 211     await cleanupTests(servers)
 212   })
 213 })